pax_global_header00006660000000000000000000000064150664657740014536gustar00rootroot0000000000000052 comment=752e3f3b1239f5e444ef397c2ac16f0b3e08036d SPIRV-LLVM-Translator-15.0.17/000077500000000000000000000000001506646577400155135ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/.clang-format000066400000000000000000000000241506646577400200620ustar00rootroot00000000000000BasedOnStyle: LLVM SPIRV-LLVM-Translator-15.0.17/.clang-tidy000066400000000000000000000017761506646577400175620ustar00rootroot00000000000000Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-no-recursion,-misc-unused-parameters,readability-identifier-naming,-llvm-header-guard' WarningsAsErrors: 'llvm-*,misc-*,-misc-no-recursion,-misc-unused-parameters,readability-identifier-naming,-llvm-header-guard' CheckOptions: - key: readability-identifier-naming.ClassCase value: CamelCase - key: readability-identifier-naming.EnumCase value: CamelCase - key: readability-identifier-naming.FunctionCase value: camelBack - key: readability-identifier-naming.MemberCase value: CamelCase - key: readability-identifier-naming.ParameterCase value: CamelCase - key: readability-identifier-naming.UnionCase value: CamelCase - key: readability-identifier-naming.VariableCase value: CamelCase - key: llvm-namespace-comment.ShortNamespaceLines value: '25' SPIRV-LLVM-Translator-15.0.17/.github/000077500000000000000000000000001506646577400170535ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/.github/workflows/000077500000000000000000000000001506646577400211105ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/.github/workflows/check-code-style.yml000066400000000000000000000145601506646577400247640ustar00rootroot00000000000000# This workflow is intended to check if PR conforms with coding standards used # in the project. # # Documentation for GitHub Actions: # [workflow-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions # [context-and-expression-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions # [workflow-commands]: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions name: Check code style on: pull_request: branches: - main - llvm_release_* paths-ignore: # no need to check formatting for: - 'docs/**' # documentation - 'test/**' # tests - '**.md' # README - '**.txt' # CMakeLists.txt - '**/check-**-build.yml' # Other workflows env: # We need compile command database in order to perform clang-tidy check. So, # in order to perform configure step we need to setup llvm-dev package. This # env variable used to specify desired version of it LLVM_VERSION: 15 jobs: clang-format-and-tidy: name: clang-format & clang-tidy runs-on: ubuntu-22.04 steps: - name: Checkout sources uses: actions/checkout@v2 with: # In order to gather diff from PR we need to fetch not only the latest # commit. Depth of 2 is enough, because GitHub Actions supply us with # merge commit as {{ github.sha }}, i.e. the second commit is a merge # base between target branch and PR fetch-depth: 2 - name: Gather list of changes id: gather-list-of-changes run: | git diff -U0 --no-color ${{ github.sha }}^ -- include lib \ ':(exclude)include/LLVMSPIRVExtensions.inc' \ ':(exclude)lib/SPIRV/libSPIRV/SPIRVErrorEnum.h' \ ':(exclude)lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h' \ ':(exclude)lib/SPIRV/libSPIRV/SPIRVOpCodeEnumInternal.h' \ > diff-to-inspect.txt if [ -s diff-to-inspect.txt ]; then # Here we set an output of our step, which is used later to either # perform or skip further steps, i.e. there is no sense to install # clang-format if PR hasn't changed .cpp files at all # See [workflow-commands] for reference echo '::set-output name=HAS_CHANGES::true' fi - name: Install dependencies if: ${{ steps.gather-list-of-changes.outputs.HAS_CHANGES }} run: | # clang-tidy requires compile command database in order to be properly # launched, so, we need to setup llvm package to perform cmake # configuration step to generate that database curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo apt-key add - echo "deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee -a /etc/apt/sources.list sudo apt-get update sudo apt-get install -yqq \ clang-format-${{ env.LLVM_VERSION }} clang-tidy-${{ env.LLVM_VERSION }} \ llvm-${{ env.LLVM_VERSION }}-dev libomp-${{ env.LLVM_VERSION }}-dev \ mlir-${{ env.LLVM_VERSION }}-tools \ - name: Generate compile command database if: ${{ steps.gather-list-of-changes.outputs.HAS_CHANGES }} run: | mkdir build && cd build cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" ${{ github.workspace }} - name: Run clang-format if: ${{ steps.gather-list-of-changes.outputs.HAS_CHANGES }} id: run-clang-format run: | cat diff-to-inspect.txt | /usr/share/clang/clang-format-${{ env.LLVM_VERSION }}/clang-format-diff.py -p1 > clang-format.patch if [ -s clang-format.patch ]; then echo "clang-format found incorrectly formatted code:" cat clang-format.patch; exit 1; else rm clang-format.patch # to avoid uploading empty file fi - name: Run clang-tidy # By some reason, GitHub Actions automatically include "success()" # expression into an "if" statement if it doesn't contain any of job # status check functions. This is why this and following steps has # "always()" and "failure()" in "if" conditions. # See "Job status check functions" in [context-and-expression-syntax] if: ${{ always() && steps.gather-list-of-changes.outputs.HAS_CHANGES }} id: run-clang-tidy run: | cat diff-to-inspect.txt | /usr/lib/llvm-${{ env.LLVM_VERSION }}/share/clang/clang-tidy-diff.py \ -p1 -clang-tidy-binary clang-tidy-${{ env.LLVM_VERSION }} -quiet \ -path ${{ github.workspace}}/build > clang-tidy.log 2>/dev/null # By some reason, clang-tidy log contains tons of extra empty lines, # that confuse the check below sed -i '/^$/d' clang-tidy.log if [ -s clang-tidy.log ]; then if ! grep -q "No relevant changes found." clang-tidy.log; then # Emit annotations while read -r line; do type="error" if [[ $line == *"warning:"* ]]; then type="warning" elif [[ $line == *"error:"* ]]; then type="error" else continue fi absolute_path=$(echo $line | grep -Po "^[\w\d-./]+(?=:)") relative_path=${absolute_path##"${{ github.workspace }}"} line_number=$(echo $line | grep -Po "(?<=:)\d+(?=:\d)") message=$(echo $line | grep -Po "(?<=${type}: ).*$") # see [workflow-commands] for documentation echo "::${type} file=${relative_path},line=${line_number}::${message}" done < clang-tidy.log echo "clang-tidy found incorrectly written code:" cat clang-tidy.log exit 1 else rm clang-tidy.log # to avoid uploading empty file fi fi - name: Upload patch with clang-format fixes uses: actions/upload-artifact@v4 if: ${{ failure() && steps.run-clang-format.outcome == 'failure' }} with: name: clang-format.patch path: clang-format.patch if-no-files-found: ignore - name: Upload clang-tidy log uses: actions/upload-artifact@v4 if: ${{ failure() && steps.run-clang-tidy.outcome == 'failure' }} with: name: clang-tidy.log path: clang-tidy.log if-no-files-found: ignore SPIRV-LLVM-Translator-15.0.17/.github/workflows/check-in-tree-build.yml000066400000000000000000000174151506646577400253560ustar00rootroot00000000000000# This workflow is intended to check that in-tree build of the translator is # healthy and all tests pass. It is used in pre-commits and nightly builds. # # Documentation for GitHub Actions: # [workflow-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions # [context-and-expression-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions name: In-tree build & tests on: push: branches: # This check is expensive; do not run it after pushes to llvm_release_* - main paths-ignore: # no need to check build for: - 'docs/**' # documentation - '**.md' # README - '**/check-code-style.yml' # check-code-style workflow - '**/check-out-of-tree-build.yml' # check-out-of-tree-build workflow pull_request: branches: - main - llvm_release_* paths-ignore: # no need to check build for: - 'docs/**' # documentation - '**.md' # README - '**/check-code-style.yml' # check-code-style workflow - '**/check-out-of-tree-build.yml' # check-out-of-tree-build workflow schedule: # Ideally, we might want to simplify our regular nightly build as we # probably don't need every configuration to be built every day: most of # them are only necessary in pre-commits to avoid breakages - cron: 0 0 * * * env: LLVM_VERSION: 15 jobs: build_and_test_linux: name: Linux strategy: matrix: build_type: [Release, Debug] shared_libs: [NoSharedLibs] include: - build_type: Release shared_libs: EnableSharedLibs fail-fast: false runs-on: ubuntu-22.04 steps: - name: Install dependencies run: | curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo apt-key add - curl -L "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo apt-key add - echo "deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee -a /etc/apt/sources.list echo "deb https://packages.lunarg.com/vulkan jammy main" | sudo tee -a /etc/apt/sources.list sudo apt-get update sudo apt-get -yq --no-install-suggests --no-install-recommends install \ clang-${{ env.LLVM_VERSION }} \ spirv-tools # Linux systems in GitHub Actions already have older versions of clang # pre-installed. Make sure to override these with the relevant version. sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.LLVM_VERSION }} 1000 - name: Checkout LLVM sources uses: actions/checkout@v2 with: repository: llvm/llvm-project ref: release/15.x path: llvm-project - name: Checkout the translator sources uses: actions/checkout@v2 with: path: llvm-project/llvm/projects/SPIRV-LLVM-Translator - name: Get tag for SPIR-V Headers id: spirv-headers-tag run: | echo "spirv_headers_tag=$(cat llvm-project/llvm/projects/SPIRV-LLVM-Translator/spirv-headers-tag.conf)" >> $GITHUB_ENV - name: Checkout SPIR-V Headers uses: actions/checkout@v2 with: repository: KhronosGroup/SPIRV-Headers ref: ${{ env.spirv_headers_tag }} path: llvm-project/llvm/projects/SPIRV-Headers - name: Configure run: | mkdir build && cd build # ON/OFF specifically weren't used as a values for shared_libs matrix # field to improve usability of PR page: instead of (Release, ON) a # job will be displayed as (Release, EnableSharedLibs) SHARED_LIBS=OFF if [[ "${{ matrix.shared_libs }}" == "EnableSharedLibs" ]]; then SHARED_LIBS=ON fi cmake ${{ github.workspace }}/llvm-project/llvm \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DBUILD_SHARED_LIBS=${SHARED_LIBS} \ -DLLVM_TARGETS_TO_BUILD="X86" \ -DSPIRV_SKIP_CLANG_BUILD=ON \ -DSPIRV_SKIP_DEBUG_INFO_TESTS=ON \ -DLLVM_LIT_ARGS="-sv --no-progress-bar" \ -G "Unix Makefiles" - name: Build run: | cd build make llvm-spirv -j2 - name: Build tests & test run: | cd build make check-llvm-spirv -j2 build_windows: name: Windows strategy: matrix: build_type: [Release] fail-fast: false runs-on: windows-latest steps: - name: Checkout LLVM sources uses: actions/checkout@v2 with: repository: llvm/llvm-project ref: release/15.x path: llvm-project - name: Checkout the translator sources uses: actions/checkout@v2 with: path: llvm-project\\llvm\\projects\\SPIRV-LLVM-Translator - name: Get tag for SPIR-V Headers id: spirv-headers-tag run: | echo "spirv_headers_tag=$(type llvm-project\\llvm\\projects\\SPIRV-LLVM-Translator\\spirv-headers-tag.conf)" >> $GITHUB_ENV - name: Checkout SPIR-V Headers uses: actions/checkout@v2 with: repository: KhronosGroup/SPIRV-Headers ref: ${{ env.spirv_headers_tag }} path: llvm-project\\llvm\\projects\\SPIRV-Headers - name: Configure shell: bash run: | mkdir build && cd build cmake ..\\llvm-project\\llvm \ -Thost=x64 \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_TARGETS_TO_BUILD="X86" \ -DSPIRV_SKIP_CLANG_BUILD=ON \ -DSPIRV_SKIP_DEBUG_INFO_TESTS=ON \ -DLLVM_LIT_ARGS="-sv --no-progress-bar" - name: Build shell: bash run: | cd build cmake --build . --config ${{ matrix.build_type }} --target llvm-spirv -j2 # FIXME: Testing is disabled at the moment as it requires clang to be present # - name: Build tests & test # shell: bash # run: | # cd build # cmake --build . --config Release --target check-llvm-spirv -j2 build_and_test_macosx: name: macOS strategy: matrix: build_type: [Release] fail-fast: false runs-on: macos-latest continue-on-error: true steps: - name: Checkout LLVM sources uses: actions/checkout@v2 with: repository: llvm/llvm-project ref: release/15.x path: llvm-project - name: Checkout the translator sources uses: actions/checkout@v2 with: path: llvm-project/llvm/projects/SPIRV-LLVM-Translator - name: Get tag for SPIR-V Headers id: spirv-headers-tag run: | echo "spirv_headers_tag=$(cat llvm-project/llvm/projects/SPIRV-LLVM-Translator/spirv-headers-tag.conf)" >> $GITHUB_ENV - name: Checkout SPIR-V Headers uses: actions/checkout@v2 with: repository: KhronosGroup/SPIRV-Headers ref: ${{ env.spirv_headers_tag }} path: llvm-project/llvm/projects/SPIRV-Headers - name: Configure run: | mkdir build && cd build cmake ${{ github.workspace }}/llvm-project/llvm \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DLLVM_TARGETS_TO_BUILD="X86" \ -DSPIRV_SKIP_CLANG_BUILD=ON \ -DSPIRV_SKIP_DEBUG_INFO_TESTS=ON \ -DLLVM_LIT_ARGS="-sv --no-progress-bar" \ -G "Unix Makefiles" - name: Build run: | cd build make llvm-spirv -j2 # FIXME: Testing is disabled at the moment as it requires clang to be present # - name: Build tests & test # run: | # cd build # make check-llvm-spirv -j2 SPIRV-LLVM-Translator-15.0.17/.github/workflows/check-out-of-tree-build.yml000066400000000000000000000072241506646577400261560ustar00rootroot00000000000000# This workflow is intended to check that out-of-tree build of the translator is # healthy and all tests pass. It is used in pre-commits and nightly builds. # # Documentation for GitHub Actions: # [workflow-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions # [context-and-expression-syntax]: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions name: Out-of-tree build & tests on: push: branches: - main - llvm_release_* paths-ignore: # no need to check build for: - 'docs/**' # documentation - '**.md' # README - '**/check-code-style.yml' # check-code-style workflow - '**/check-in-tree-build.yml' # check-in-tree-build workflow pull_request: branches: - main - llvm_release_* paths-ignore: # no need to check build for: - 'docs/**' # documentation - '**.md' # README - '**/check-code-style.yml' # check-code-style workflow - '**/check-in-tree-build.yml' # check-in-tree-build workflow schedule: - cron: 0 0 * * * env: LLVM_VERSION: 15 jobs: build_and_test: name: Linux strategy: matrix: build_type: [Release, Debug] fail-fast: false runs-on: ubuntu-22.04 steps: - name: Install dependencies run: | curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo apt-key add - curl -L "https://packages.lunarg.com/lunarg-signing-key-pub.asc" | sudo apt-key add - echo "deb https://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | sudo tee -a /etc/apt/sources.list echo "deb https://packages.lunarg.com/vulkan jammy main" | sudo tee -a /etc/apt/sources.list sudo apt-get update sudo apt-get -yq --no-install-suggests --no-install-recommends install \ clang-${{ env.LLVM_VERSION }} \ llvm-${{ env.LLVM_VERSION }}-dev \ libomp-${{ env.LLVM_VERSION }}-dev \ llvm-${{ env.LLVM_VERSION }}-tools \ mlir-${{ env.LLVM_VERSION }}-tools \ spirv-tools # Linux systems in GitHub Actions already have older versions of clang # pre-installed. Make sure to override these with the relevant version. sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.LLVM_VERSION }} 1000 - name: Checkout the translator sources uses: actions/checkout@v2 with: path: SPIRV-LLVM-Translator - name: Get tag for SPIR-V Headers id: spirv-headers-tag run: | echo "spirv_headers_tag=$(cat SPIRV-LLVM-Translator/spirv-headers-tag.conf)" >> $GITHUB_ENV - name: Checkout SPIR-V Headers uses: actions/checkout@v2 with: repository: KhronosGroup/SPIRV-Headers ref: ${{ env.spirv_headers_tag }} path: SPIRV-Headers - name: Configure run: | mkdir build && cd build cmake ${{ github.workspace }}/SPIRV-LLVM-Translator \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DCMAKE_CXX_FLAGS="-Werror -Wno-error=deprecated-declarations" \ -DLLVM_INCLUDE_TESTS=ON \ -DLLVM_EXTERNAL_LIT="/usr/lib/llvm-${{ env.LLVM_VERSION }}/build/utils/lit/lit.py" \ -DLLVM_EXTERNAL_PROJECTS="SPIRV-Headers" \ -DLLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR=${{ github.workspace }}/SPIRV-Headers \ -G "Unix Makefiles" - name: Build run: | cd build make llvm-spirv -j2 - name: Build tests & test run: | cd build make check-llvm-spirv -j2 SPIRV-LLVM-Translator-15.0.17/.gitignore000066400000000000000000000021201506646577400174760ustar00rootroot00000000000000#==============================================================================# # This file specifies intentionally untracked files that git should ignore. # See: http://www.kernel.org/pub/software/scm/git/docs/gitignore.html #==============================================================================# build/ #==============================================================================# # File extensions to be ignored anywhere in the tree. #==============================================================================# # Temp files created by most text editors. *~ # Merge files created by git. *.orig # Byte compiled python modules. *.pyc # vim swap files .*.sw? .sw? #OS X specific files. .DS_store #==============================================================================# # Explicit files to ignore (only matches one). #==============================================================================# # Various tag programs /tags /TAGS /GPATH /GRTAGS /GSYMS /GTAGS .gitusers autom4te.cache cscope.files cscope.out autoconf/aclocal.m4 autoconf/autom4te.cache compile_commands.json SPIRV-LLVM-Translator-15.0.17/CMakeLists.txt000066400000000000000000000105071506646577400202560ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.13.4) if(NOT DEFINED BASE_LLVM_VERSION) set (BASE_LLVM_VERSION 15.0.0) endif(NOT DEFINED BASE_LLVM_VERSION) set(LLVM_SPIRV_VERSION ${BASE_LLVM_VERSION}.0) include(FetchContent) include(FindPkgConfig) option(LLVM_SPIRV_INCLUDE_TESTS "Generate build targets for the llvm-spirv lit tests." ${LLVM_INCLUDE_TESTS}) if (NOT DEFINED LLVM_SPIRV_BUILD_EXTERNAL) # check if we build inside llvm or not if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(LLVM_SPIRV_BUILD_EXTERNAL YES) endif(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) endif (NOT DEFINED LLVM_SPIRV_BUILD_EXTERNAL) if(LLVM_SPIRV_BUILD_EXTERNAL) # Make sure llvm-spirv gets built when building outside the llvm tree. set(LLVM_BUILD_TOOLS ON) endif(LLVM_SPIRV_BUILD_EXTERNAL) # Download spirv.hpp from the official SPIRV-Headers repository. # One can skip this step by manually setting # LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR path. if(NOT DEFINED LLVM_TOOL_SPIRV_HEADERS_BUILD AND NOT DEFINED LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR) set(LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/SPIRV-Headers") message(STATUS "SPIR-V Headers location is not specified. Will try to download spirv.hpp from https://github.com/KhronosGroup/SPIRV-Headers into ${LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR}") file(READ spirv-headers-tag.conf SPIRV_HEADERS_TAG) # Strip the potential trailing newline from tag string(STRIP "${SPIRV_HEADERS_TAG}" SPIRV_HEADERS_TAG) FetchContent_Declare(spirv-headers GIT_REPOSITORY https://github.com/KhronosGroup/SPIRV-Headers.git GIT_TAG ${SPIRV_HEADERS_TAG} SOURCE_DIR ${LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR} ) FetchContent_MakeAvailable(spirv-headers) else() if(NOT DEFINED LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR) # This means LLVM_TOOL_SPIRV_HEADERS_BUILD is defined, therefore # SPIRV-Headers exist as a subproject. set(LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR "${CMAKE_SOURCE_DIR}/projects/SPIRV-Headers") if(NOT EXISTS ${LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR}) message(FATAL_ERROR "No location specified for SPIRV-Headers. Try setting the LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR path or put the project into the llvm/projects folder under the name 'SPIRV-Headers'") endif() endif() message(STATUS "Using SPIR-V Headers from ${LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR}") endif() if(LLVM_SPIRV_BUILD_EXTERNAL) project(LLVM_SPIRV VERSION ${LLVM_SPIRV_VERSION} LANGUAGES CXX C ) set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) if(LLVM_SPIRV_INCLUDE_TESTS) set(LLVM_TEST_COMPONENTS llvm-as llvm-dis ) endif(LLVM_SPIRV_INCLUDE_TESTS) find_package(LLVM ${BASE_LLVM_VERSION} REQUIRED COMPONENTS Analysis BitReader BitWriter CodeGen Core Passes Support TransformUtils ${LLVM_TEST_COMPONENTS} ) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${LLVM_CMAKE_DIR} ) include(AddLLVM) include(HandleLLVMOptions) message(STATUS "Found LLVM: ${LLVM_VERSION}") option(CCACHE_ALLOWED "allow use of ccache" TRUE) find_program(CCACHE_EXE_FOUND ccache) if(CCACHE_EXE_FOUND AND CCACHE_ALLOWED) message(STATUS "Found ccache: ${CCACHE_EXE_FOUND}") set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) endif() endif() set(LLVM_SPIRV_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include) pkg_search_module(SPIRV_TOOLS SPIRV-Tools) if(NOT SPIRV_TOOLS_FOUND) message(STATUS "SPIRV-Tools not found; project will be built without " "--spirv-tools-dis support.") endif(NOT SPIRV_TOOLS_FOUND) add_subdirectory(lib/SPIRV) add_subdirectory(tools/llvm-spirv) if(LLVM_SPIRV_INCLUDE_TESTS) add_subdirectory(test) endif(LLVM_SPIRV_INCLUDE_TESTS) install( FILES ${LLVM_SPIRV_INCLUDE_DIRS}/LLVMSPIRVLib.h ${LLVM_SPIRV_INCLUDE_DIRS}/LLVMSPIRVOpts.h ${LLVM_SPIRV_INCLUDE_DIRS}/LLVMSPIRVExtensions.inc DESTINATION ${CMAKE_INSTALL_PREFIX}/include/LLVMSPIRVLib ) configure_file(LLVMSPIRVLib.pc.in ${CMAKE_BINARY_DIR}/LLVMSPIRVLib.pc @ONLY) install( FILES ${CMAKE_BINARY_DIR}/LLVMSPIRVLib.pc DESTINATION ${CMAKE_INSTALL_PREFIX}/lib${LLVM_LIBDIR_SUFFIX}/pkgconfig ) SPIRV-LLVM-Translator-15.0.17/CODE_OF_CONDUCT.md000066400000000000000000000004301506646577400203070ustar00rootroot00000000000000A reminder that this issue tracker is managed by the Khronos Group. Interactions here should follow the Khronos Code of Conduct (https://www.khronos.org/developers/code-of-conduct), which prohibits aggressive or derogatory language. Please keep the discussion friendly and civil. SPIRV-LLVM-Translator-15.0.17/CONTRIBUTING.md000066400000000000000000000137561506646577400177600ustar00rootroot00000000000000# Contribution guidelines ## If you have found a bug or would like to see a new feature Please reach us by creating a [new issue]. Your bug report should include a proper description and steps to reproduce: - attach the LLVM BC or SPV file you are trying to translate and the command you launch - any backtrace in case of crashes would be helpful - please describe what goes wrong or what is unexpected during translation For feature requests, please describe the feature you would like to see implemented in the translator. [new issue]: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/new ## If you would like to contribute your change Please open a [pull request]. If you are not sure whether your changes are correct, you can either mark it as [draft] or create an issue to discuss the problem and possible ways to fix it prior to publishing a PR. It is okay to have several commits in the PR, but each of them should be buildable and tests should pass. Maintainers can squash several commits into a single one for you during merge, but if you would like to see several commits in the git history, please let us know in PR description/comments so maintainers will rebase your PR instead of squashing it. Each functional change (new feature or bug fix) must be supplied with corresponding tests. See [#testing-guidelines] for more information about testing. NFC (non-functional change) PRs can be accepted without new tests. Code changes should follow coding standards, which are inherited from [LLVM Coding Standards]. Compliance of your code is checked automatically using Travis CI. See [clang-format] and [clang-tidy] configs for more details about coding standards. ## How to add an extension First of all please make sure you have added a link to the specification for the extension in your PR. Then to add definitions of new Op Codes you shall modify [spirv.hpp], which is an external dependency for this project. To do so, you should add new definitions to [json grammar file], rebuild the header following the [instructions] in [SPIR-V Headers repository] and push your changes for review, i.e. make a PR. Once the PR is merged, a new spirv.hpp will have to be downloaded during build of the translator; make sure to update the hash for SPIRV-Headers in [spirv-headers-tag.conf] so that tokens from your extension can be visible to the translator build. It's highly recommended to add the definitions to [SPIR-V Headers repository] first, but if you don't want to bring it there yet, you can define new Op Codes in the [internal SPIR-V header file]. For local testing you can copy your spirv.hpp variant to `/include/spirv/unified1` and/or modify it there. See [README.md](README.md#configuring-spir-v-headers) for build instructions that should be employed with such modifications. ### Conditions to merge a PR In order to get your PR merged, the following conditions must be met: - If you are a first-time contributor, you have to sign the [Contributor License Agreement]. Corresponding link and instructions will be automatically posted into your PR. - [Travis CI testing] jobs must pass on your PR: this includes functional testing and checking for complying with coding standards. - You need to get approval from at least one contributor with merge rights. As a contributor, you should expect that even an approved PR might still be left open for a few days: this is needed, because the translator is being developed by different vendors and individuals and we need to ensure that each interested party is able to react to new changes and provide feedback. Information below is a guideline for repo maintainers and can be used by contributors to get some expectations about how long a PR has to be open before it can be merged: - For any significant change/redesign, the PR must be open for at least 5 working days, so everyone interested can step in to provide feedback, discuss direction and help to find bugs. - Ideally, there should be approvals from different vendors/individuals to get it merged, particularly for larger changes. - For regular changes/bug fixes, the PR must be open for at least 2-3 working days, so everyone interested can step in for review and provide feedback. - If the change is vendor-specific (bug fix in vendor extension implementation or new vendor-specific extension support), then it is okay to merge PR sooner. - If the change affects or might affect several interested parties, the PR must be left open for 2-3 working days and it would be good to see feedback from different vendors/inviduals before merging. - Tiny NFC changes or trivial build fixes (due to LLVM API changes) can be submitted as soon as testing is finished and PR approved - no need to wait for too long. - In general, just use common sense to wait long enough to get feedback from everyone who might be interested in the PR and don't hesitate to explicitly mention individuals who might be interested in reviewing the PR. [pull request]: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/pulls [draft]: https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests [LLVM Coding Standards]: https://llvm.org/docs/CodingStandards.html [clang-format]: [.clang-format] [clang-tidy]: [.clang-tidy] [spirv.hpp]: https://github.com/KhronosGroup/SPIRV-Headers/blob/master/include/spirv/unified1/spirv.hpp [json grammar file]: https://github.com/KhronosGroup/SPIRV-Headers/blob/master/include/spirv/unified1/spirv.core.grammar.json [instructions]: https://github.com/KhronosGroup/SPIRV-Headers#generating-headers-from-the-json-grammar-for-the-spir-v-core-instruction-set [SPIR-V Headers repository]: https://github.com/KhronosGroup/SPIRV-Headers [internal SPIR-V header file]: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/main/lib/SPIRV/libSPIRV/spirv_internal.hpp [Contributor License Agreement]: https://cla-assistant.io/KhronosGroup/SPIRV-LLVM-Translator [Travis CI testing]: https://travis-ci.org/KhronosGroup/SPIRV-LLVM-Translator SPIRV-LLVM-Translator-15.0.17/LICENSE.TXT000066400000000000000000000063211506646577400172000ustar00rootroot00000000000000============================================================================== LLVM Release License ============================================================================== University of Illinois/NCSA Open Source License Copyright (c) 2003-2014 University of Illinois at Urbana-Champaign. All rights reserved. Developed by: LLVM Team University of Illinois at Urbana-Champaign http://llvm.org Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal with the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimers. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimers in the documentation and/or other materials provided with the distribution. * Neither the names of the LLVM Team, University of Illinois at Urbana-Champaign, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE. ============================================================================== Copyrights and Licenses for Third Party Software Distributed with LLVM: ============================================================================== The LLVM software contains code written by third parties. Such software will have its own individual LICENSE.TXT file in the directory in which it appears. This file will describe the copyrights, license, and restrictions which apply to that code. The disclaimer of warranty in the University of Illinois Open Source License applies to all code in the LLVM Distribution, and nothing in any of the other licenses gives permission to use the names of the LLVM Team or the University of Illinois to endorse or promote products derived from this Software. The following pieces of software have additional or alternate copyrights, licenses, and/or restrictions: Program Directory ------- --------- Autoconf llvm/autoconf llvm/projects/ModuleMaker/autoconf Google Test llvm/utils/unittest/googletest OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} ARM contributions llvm/lib/Target/ARM/LICENSE.TXT md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h SPIRV-LLVM-Translator-15.0.17/LLVMSPIRVLib.pc.in000066400000000000000000000005241506646577400204720ustar00rootroot00000000000000prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} libdir=${prefix}/lib@LLVM_LIBDIR_SUFFIX@ includedir=${prefix}/include Name: LLVMSPIRVLib Description: LLVM/SPIR-V bi-directional translator Version: @LLVM_SPIRV_VERSION@ URL: https://github.com/KhronosGroup/SPIRV-LLVM-Translator Libs: -L${libdir} -lLLVMSPIRVLib Cflags: -I${includedir} SPIRV-LLVM-Translator-15.0.17/README.md000066400000000000000000000316311506646577400167760ustar00rootroot00000000000000# LLVM/SPIR-V Bi-Directional Translator [![Out-of-tree build & tests](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/workflows/Out-of-tree%20build%20&%20tests/badge.svg?branch=llvm_release_150&event=schedule)](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/actions?query=workflow%3A%22Out-of-tree+build+%26+tests%22+event%3Aschedule) [![In-tree build & tests](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/workflows/In-tree%20build%20&%20tests/badge.svg?branch=llvm_release_150&event=schedule)](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/actions?query=workflow%3A%22In-tree+build+%26+tests%22+event%3Aschedule) This repository contains source code for the LLVM/SPIR-V Bi-Directional Translator, a library and tool for translation between LLVM IR and [SPIR-V](https://www.khronos.org/registry/spir-v/). The LLVM/SPIR-V Bi-Directional Translator is open source software. You may freely distribute it under the terms of the license agreement found in LICENSE.txt. ## Directory Structure The files/directories related to the translator: * [include/LLVMSPIRVLib.h](include/LLVMSPIRVLib.h) - header file * [lib/SPIRV](lib/SPIRV) - library for SPIR-V in-memory representation, decoder/encoder and LLVM/SPIR-V translator * [tools/llvm-spirv](tools/llvm-spirv) - command line utility for translating between LLVM bitcode and SPIR-V binary ## Build Instructions The `main` branch of this repo is aimed to be buildable with the latest LLVM `main` revision. ### Build with pre-installed LLVM The translator can be built with the latest(nightly) package of LLVM. For Ubuntu and Debian systems LLVM provides repositories with nightly builds at http://apt.llvm.org/. For example the latest package for Ubuntu 16.04 can be installed with the following commands: ``` wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main" sudo apt-get update sudo apt-get install llvm-15-dev llvm-15-tools clang-15 libclang-15-dev ``` The installed version of LLVM will be used by default for out-of-tree build of the translator. ``` git clone https://github.com/KhronosGroup/SPIRV-LLVM-Translator.git mkdir SPIRV-LLVM-Translator/build && cd SPIRV-LLVM-Translator/build cmake .. make llvm-spirv -j`nproc` ``` ### Build with pre-built LLVM If you have a custom build (based on the latest version) of LLVM libraries you can link the translator against it. ``` git clone https://github.com/KhronosGroup/SPIRV-LLVM-Translator.git mkdir SPIRV-LLVM-Translator/build && cd SPIRV-LLVM-Translator/build cmake .. -DLLVM_DIR=/lib/cmake/llvm/ make llvm-spirv -j`nproc` ``` If the translator is used as part of another CMake project, you will need to define `LLVM_SPIRV_BUILD_EXTERNAL`: ``` cmake .. -DLLVM_DIR=/lib/cmake/llvm/ -DLLVM_SPIRV_BUILD_EXTERNAL=YES ``` Where `llvm_build_dir` is the LLVM build directory. ### LLVM in-tree build The translator can be built as a regular LLVM subproject. To do that you need to clone it into the `llvm/projects` or `llvm/tools` directory. ``` git clone https://github.com/llvm/llvm-project.git cd llvm-project/llvm/projects git clone https://github.com/KhronosGroup/SPIRV-LLVM-Translator.git ``` Run (or re-run) cmake as usual for LLVM. After that you should have `llvm-spirv` and `check-llvm-spirv` targets available. ``` mkdir llvm-project/build && cd llvm-project/build cmake ../llvm -DLLVM_ENABLE_PROJECTS="clang" make llvm-spirv -j`nproc` ``` Note on enabling the `clang` project: there are tests in the translator that depend on `clang` binary, which makes clang a required dependency (search for `LLVM_SPIRV_TEST_DEPS` in [test/CMakeLists.txt](test/CMakeLists.txt)) for `check-llvm-spirv` target. Building clang from sources takes time and resources and it can be avoided: - if you are not interested in launching unit-tests for the translator after build, you can disable generation of test targets by passing `-DLLVM_SPIRV_INCLUDE_TESTS=OFF` option. - if you are interested in launching unit-tests, but don't want to build clang you can pass `-DSPIRV_SKIP_CLANG_BUILD` cmake option to avoid adding `clang` as dependency for `check-llvm-spirv` target. However, LIT will search for `clang` binary when tests are launched and it should be available at this point. - building and testing completely without `clang` is not supported at the moment, see [KhronosGroup/SPIRV-LLVM-Translator#477](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/477) to track progress, discuss and contribute. ### Build with SPIRV-Tools The translator can use [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools) to generate assembly with widely adopted syntax. If SPIRV-Tools have been installed prior to the build it will be detected and used automatically. However it is also possible to enable use of SPIRV-Tools from a custom location using the following instructions: 1. Checkout, build and install SPIRV-Tools using [the following instructions](https://github.com/KhronosGroup/SPIRV-Tools#build). Example using CMake with Ninja: ``` cmake -G Ninja -DCMAKE_INSTALL_PREFIX= ninja install ``` 2. Point pkg-config to the SPIR-V tools installation when configuring the translator by setting `PKG_CONFIG_PATH=/lib/pkgconfig/` variable before the cmake line invocation. Example: ``` PKG_CONFIG_PATH=/lib/pkgconfig/ cmake ``` To verify the SPIR-V Tools integration in the translator build, run the following line ``` llvm-spirv --spirv-tools-dis input.bc -o - ``` The output should be printed in the standard assembly syntax. ## Configuring SPIR-V Headers The translator build is dependent on the official Khronos header file `spirv.hpp` that maps SPIR-V extensions, decorations, instructions, etc. onto numeric tokens. The official header version is available at [KhronosGroup/SPIRV-Headers](https://github.com/KhronosGroup/SPIRV-Headers). There are several options for accessing the header file: - By default, the header file repository will be downloaded from Khronos Group GitHub and put into `/SPIRV-Headers`. - If you are building the translator in-tree, you can manually download the SPIR-V Headers repo into `llvm/projects` - this location will be automatically picked up by the LLVM build scripts. Make sure the folder retains its default naming in that of `SPIRV-Headers`. - Any build type can also use an external installation of SPIR-V Headers - if you have the headers downloaded somewhere in your system and want to use that version, simply extend your CMake command with `-DLLVM_EXTERNAL_PROJECTS="SPIRV-Headers" -DLLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR=`. ## Test instructions All tests related to the translator are placed in the [test](test) directory. A number of the tests require spirv-as (part of SPIR-V Tools) to run, but the remainder of the tests can still be run without this. Optionally the tests can make use of spirv-val (part of SPIRV-Tools) in order to validate the generated SPIR-V against the official SPIR-V specification. In case tests are failing due to SPIRV-Tools not supporting certain SPIR-V features, please get an updated package. The `PKG_CONFIG_PATH` environmental variable can be used to let cmake point to a custom installation. Execute the following command inside the build directory to run translator tests: ``` make test ``` This requires that the `-DLLVM_SPIRV_INCLUDE_TESTS=ON` argument is passed to CMake during the build step. Additionally, `-DLLVM_EXTERNAL_LIT="/usr/lib/llvm-15/build/utils/lit/lit.py"` is needed when building with a pre-installed version of LLVM. The translator test suite can be disabled by passing `-DLLVM_SPIRV_INCLUDE_TESTS=OFF` to CMake. ## Run Instructions for `llvm-spirv` To translate between LLVM IR and SPIR-V: 1. Execute the following command to translate `input.bc` to `input.spv` ``` llvm-spirv input.bc ``` 2. Execute the following command to translate `input.spv` to `input.bc` ``` llvm-spirv -r input.spv ``` Recommended options: * `-spirv-ocl-builtins-version` - to specify target version of OpenCL builtins to translate to (default CL1.2) 3. Other options accepted by `llvm-spirv` * `-o file_name` - to specify output name * `-spirv-debug` - output debugging information * `-spirv-text` - read/write SPIR-V in an internal textual format for debugging purpose. The textual format is not defined by SPIR-V spec. * `--spirv-tools-dis` - print SPIR-V assembly in SPIRV-Tools format. Only available on [builds with SPIRV-Tools](#build-with-spirv-tools). * `-help` - to see full list of options Translation from LLVM IR to SPIR-V and then back to LLVM IR is not guaranteed to produce the original LLVM IR. In particular, LLVM intrinsic call instructions may get replaced by function calls to OpenCL builtins and metadata may be dropped. ### Handling SPIR-V versions generated by the translator There is one option to control the behavior of the translator with respect to the version of the SPIR-V file which is being generated/consumed. * `-spirv-max-version=` - this option allows restricting the SPIRV-LLVM-Translator **not** to generate a SPIR-V with a version which is higher than the one specified via this option. If the `-r` option was also specified, the SPIRV-LLVM-Translator will reject the input file and emit an error if the SPIR-V version in it is higher than one specified via this option. Allowed values are `1.0`, `1.1`, `1.2`, `1.3`, `1.4`, `1.5` and `1.6`. More information can be found in [SPIR-V versions and extensions handling](docs/SPIRVVersionsAndExtensionsHandling.rst) ### Handling SPIR-V extensions generated by the translator By default, during SPIR-V generation, the translator doesn't use any extensions. However, during SPIR-V consumption, the translator accepts input files that use any known extensions. If certain extensions are required to be enabled or disabled, the following command line option can be used: * ``--spirv-ext=`` - this options allows controlling which extensions are allowed/disallowed Valid value for this option is comma-separated list of extension names prefixed with ``+`` or ``-`` - plus means allow to use extension, minus means disallow to use extension. There is one more special value which can be used as extension name in this option: ``all`` - it affects all extension which are known to the translator. If ``--spirv-ext`` contains the name of an extension which is not known for the translator, it will emit an error. More information can be found in [SPIR-V versions and extensions handling](docs/SPIRVVersionsAndExtensionsHandling.rst) ## Branching strategy Code on the main branch in this repository is intended to be compatible with the main branch of the [llvm](https://github.com/llvm/llvm-project) project. That is, for an OpenCL kernel compiled to llvm bitcode by the latest git revision of Clang it should be possible to translate it to SPIR-V with the llvm-spirv tool. All new development should be done on the main branch. To have versions compatible with released versions of LLVM and Clang, corresponding tags are available in this repository. For example, to build the translator with [LLVM 7.0.0](https://github.com/llvm/llvm-project/tree/llvmorg-7.0.0) one should use the [v7.0.0-1](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/v7.0.0-1) tag. The 7.x releases are maintained on the [llvm_release_70](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/tree/llvm_release_70) branch. As a general rule, commits from the main branch may be backported to the release branches as long as they do not depend on features from a later LLVM/Clang release and there are no objections from the maintainer(s). There is no guarantee that older release branches are proactively kept up to date with main, but you can request specific commits on older release branches by creating a pull request or raising an issue on GitHub. ## Deprecation of "preview extensions" In a case if a "preview extension" has to be deprecated, as the first step one should disable support for forward translation of the extension in the main branch (this will prevent new SPIR-V modules from being generated using the extension). Meanwhile support reverse translation for the extension should be continued (this retains compatibility with existing SPIR-V modules). * Addition of deprecation warning for the extension is not required, yet recommended for extensions adding instructions representable in LLVM IR only via SPIR-V friendly LLVM IR. * We encourage backporting the changes to other branches to speed up removal, but this is not required. After at least one release cycle one may remove support for reverse translation in the main branch as well, at which point support for the "preview extension" is considered removed. These are guidelines, not requirements, and we will consider exceptions on a case-by-case basis. SPIRV-LLVM-Translator-15.0.17/docs/000077500000000000000000000000001506646577400164435ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/docs/SPIRVRepresentationInLLVM.rst000066400000000000000000000602531506646577400240130ustar00rootroot00000000000000================================ SPIR-V representation in LLVM IR ================================ .. contents:: :local: Overview ======== As one of the goals of SPIR-V is to `"map easily to other IRs, including LLVM IR" `_, most of SPIR-V entities (global variables, constants, types, functions, basic blocks, instructions) have straightforward counterparts in LLVM. Therefore the focus of this document is those entities in SPIR-V which do not map to LLVM in an obvious way. These include: * SPIR-V types mapped to LLVM types * SPIR-V instructions mapped to LLVM function calls * SPIR-V extended instructions mapped to LLVM function calls * SPIR-V builtin variables mapped to LLVM function calls or LLVM global variables * SPIR-V instructions mapped to LLVM metadata * SPIR-V types mapped to LLVM opaque types * SPIR-V decorations mapped to LLVM metadata or named attributes * Additional requirements for LLVM module SPIR-V Types Mapped to LLVM Types ================================= Limited to this section, we define the following common postfix. * {Access} - Postifix indicating the access qualifier. {Access} take integer literal values which are defined by the SPIR-V spec. OpTypeImage ----------- OpTypeImage is mapped to LLVM opaque type spirv.Image._{SampledType}_{Dim}_{Depth}_{Arrayed}_{MS}_{Sampled}_{Format}_{Access} and mangled as __spirv_Image__{SampledType}_{Dim}_{Depth}_{Arrayed}_{MS}_{Sampled}_{Format}_{Access}, where * {SampledType}={float|half|int|uint|void} - Postfix indicating the sampled data type - void for unknown sampled data type * {Dim} - Postfix indicating the dimension of the image * {Depth} - Postfix indicating whether the image is a depth image * {Arrayed} - Postfix indicating whether the image is arrayed image * {MS} - Postfix indicating whether the image is multi-sampled * {Sampled} - Postfix indicating whether the image is associated with sampler * {Format} - Postfix indicating the image format Postfixes {Dim}, {Depth}, {Arrayed}, {MS}, {Sampled} and {Format} take integer literal values which are defined by the SPIR-V spec. OpTypeSampledImage ------------------ OpTypeSampledImage is mapped to LLVM opaque type spirv.SampledImage._{Postfixes} and mangled as __spirv_SampledImage__{Postfixes}, where {Postfixes} are the same as the postfixes of the original image type, as defined above in this section. OpTypePipe ---------- OpTypePipe is mapped to LLVM opaque type spirv.Pipe._{Access} and mangled as __spirv_Pipe__{Access}. Other SPIR-V Types ------------------ * OpTypeEvent * OpTypeDeviceEvent * OpTypeReserveId * OpTypeQueue * OpTypeSampler * OpTypePipeStorage (SPIR-V 1.1) The above SPIR-V types are mapped to LLVM opaque type spirv.{TypeName} and mangled as __spirv_{TypeName}, where {TypeName} is the name of the SPIR-V type with "OpType" removed, e.g., OpTypeEvent is mapped to spirv.Event and mangled as __spirv_Event. Address spaces -------------- The following `SPIR-V storage classes `_ are naturally represented as LLVM IR address spaces with the following mapping: ==================== ==================================== SPIR-V storage class LLVM IR address space ==================== ==================================== ``Function`` No address space or ``addrspace(0)`` ``CrossWorkgroup`` ``addrspace(1)`` ``UniformConstant`` ``addrspace(2)`` ``Workgroup`` ``addrspace(3)`` ``Generic`` ``addrspace(4)`` ==================== ==================================== SPIR-V extensions are allowed to add new storage classes. For example, SPV_INTEL_usm_storage_classes extension adds ``DeviceOnlyINTEL`` and ``HostOnlyINTEL`` storage classes which are mapped to ``addrspace(5)`` and ``addrspace(6)`` respectively. SPIR-V Instructions Mapped to LLVM Function Calls ================================================= Some SPIR-V instructions which can be included in basic blocks do not have corresponding LLVM instructions or intrinsics. These SPIR-V instructions are represented by function calls in LLVM. The function corresponding to a SPIR-V instruction is termed SPIR-V builtin function and its name is `IA64 mangled `_ with extensions for SPIR-V specific types. The unmangled name of a SPIR-V builtin function follows the convention .. code-block:: c __spirv_{OpCodeName}{_OptionalPostfixes} where {OpCodeName} is the op code name of the SPIR-V instructions without the "Op" prefix, e.g. EnqueueKernel. {OptionalPostfixes} are optional postfixes to specify decorations for the SPIR-V instruction. The SPIR-V op code name and each postfix does not contain "_". SPIR-V builtin functions accepts all argument types accepted by the corresponding SPIR-V instructions. The literal operands of extended instruction are mapped to function call arguments with type i32. Optional Postfixes for SPIR-V Builtin Function Names ---------------------------------------------------- SPIR-V builtin functions corresponding to the following SPIR-V instructions are postfixed following the order specified as below: * Instructions having identical argument types but different return types are postfixed with "_R{ReturnType}" where - {ReturnType} = {ScalarType}|{VectorType} - {ScalarType} = char|uchar|short|ushort|int|uint|long|ulong|half|float|double|bool - {VectorType} = {ScalarType}{2|3|4|8|16} * Instructions with saturation decoration are postfixed with "_sat" * Instructions with floating point rounding mode decoration are postfixed with "_rtp|_rtn|_rtz|_rte" SPIR-V Builtin Conversion Function Names ---------------------------------------- The unmangled names of SPIR-V builtin conversion functions follow the convention: .. code-block:: c __spirv_{ConversionOpCodeName}_R{ReturnType}{_sat}{_rtp|_rtn|_rtz|_rte} where * {ConversionOpCodeName} = ConvertFToU|ConvertFToS|ConvertUToF|ConvertUToS|UConvert|SConvert|FConvert|SatConvertSToU|SatConvertUToS SPIR-V Builtin Reinterpret / Bitcast Function Names --------------------------------------------------- The unmangled names of SPIR-V builtin reinterpret / bitcast functions follow the convention: .. code-block:: c __spirv_{BitcastOpCodeName}_R{ReturnType} SPIR-V Builtin ImageSample Function Names ---------------------------------------- The unmangled names of SPIR-V builtin ImageSample functions follow the convention: .. code-block:: c __spirv_{ImageSampleOpCodeName}_R{ReturnType} SPIR-V Builtin GenericCastToPtr Function Name ---------------------------------------- The unmangled names of SPIR-V builtin GenericCastToPtrExplicit function follow the convention: .. code-block:: c __spirv_GenericCastToPtrExplicit_To{Global|Local|Private} SPIR-V Builtin BuildNDRange Function Name ---------------------------------------- The unmangled names of SPIR-V builtin BuildNDRange functions follow the convention: .. code-block:: c __spirv_{BuildNDRange}_{1|2|3}D SPIR-V 1.1 Builtin CreatePipeFromPipeStorage Function Name ---------------------------------------- The unmangled names of SPIR-V builtin CreatePipeFromPipeStorage function follow the convention: .. code-block:: c __spirv_CreatePipeFromPipeStorage_{read|write} SPIR-V Extended Instructions Mapped to LLVM Function Calls ========================================================== SPIR-V extended instructions are mapped to LLVM function calls. The function name is IA64 mangled and the unmangled name has the format .. code-block:: c __spirv_{ExtendedInstructionSetName}_{ExtendedInstrutionName}{__OptionalPostfixes} where {ExtendedInstructionSetName} for OpenCL is "ocl". The translated functions accepts all argument types accepted by the corresponding SPIR-V instructions. The literal operands of extended instruction are mapped to function call arguments with type i32. The optional postfixes take the same format as SPIR-V builtin functions. The first postfix starts with two underscores to facilitate identification since extended instruction name may contain underscore. The remaining postfixes start with one underscore. OpenCL Extended Builtin Vector Load Function Names ---------------------------------------- The unmangled names of OpenCL extended vector load functions follow the convention: .. code-block:: c __spirv_ocl_{VectorLoadOpCodeName}__R{ReturnType} where * {VectorLoadOpCodeName} = vloadn|vload_half|vload_halfn|vloada_halfn SPIR-V Builtin Variables Mapped to LLVM Function Calls or LLVM Global Variables =============================================================================== By default each access of SPIR-V builtin variable's value is mapped to LLVM function call. The unmangled names of these functions follow the convention: .. code-block:: c __spirv_BuiltIn{VariableName} In case if SPIR-V builtin variable has vector type, the corresponding LLVM function will have an integer argument, so each access of the variable's scalar component is mapped to a function call with index argument, i.e.: .. code-block:: llvm ; For scalar variables ; SPIR-V OpDecorate %__spirv_BuiltInGlobalInvocationId BuiltIn GlobalInvocationId %13 = OpLoad %uint %__spirv_BuiltInGlobalLinearId Aligned 4 ; Will be transformed into the following LLVM IR: %0 = call spir_func i32 @_Z29__spirv_BuiltInGlobalLinearIdv() ; For vector variables ; SPIRV OpDecorate %__spirv_BuiltInGlobalInvocationId BuiltIn GlobalInvocationId %14 = OpLoad %v3ulong %__spirv_BuiltInGlobalInvocationId Aligned 32 %15 = OpCompositeExtract %ulong %14 1 ; Can be transformed into the following LLVM IR: %0 = call spir_func i64 @_Z33__spirv_BuiltInGlobalInvocationIdi(i32 1) ; However SPIRV-LLVM translator will transform it to the following pattern: %1 = call spir_func i64 @_Z33__spirv_BuiltInGlobalInvocationIdi(i32 0) %2 = insertelement <3 x i64> undef, i64 %1, i32 0 %3 = call spir_func i64 @_Z33__spirv_BuiltInGlobalInvocationIdi(i32 1) %4 = insertelement <3 x i64> %2, i64 %3, i32 1 %5 = call spir_func i64 @_Z33__spirv_BuiltInGlobalInvocationIdi(i32 2) %6 = insertelement <3 x i64> %4, i64 %5, i32 2 %7 = extractelement <3 x i64> %6, i32 1 ; In case some actions are performed with the variable's value in vector form. SPIR-V builtin variables can also be mapped to LLVM global variables with unmangled name __spirv_BuiltIn{Name}. The representation with variables is closer to SPIR-V, so it is easier to translate from SPIR-V to LLVM and back using it. Hovewer in languages like OpenCL the functionality covered by SPIR-V builtin variables is usually represented by builtin functions, so it is easier to translate from/to SPIR-V friendly IR to/from LLVM IR produced from OpenCL-like source languages. That is why both forms of mapping are supported. SPIR-V instructions mapped to LLVM metadata =========================================== SPIR-V specification allows multiple module scope instructions, whereas LLVM named metadata must be unique, so encoding of such instructions has the following format: .. code-block:: llvm !spirv. = !{!, !, ..} ! = !{, , ..} ! = !{, , ..} +--------------------+---------------------------------------------------------+ | SPIR-V instruction | LLVM IR | +====================+=========================================================+ | OpSource | .. code-block:: llvm | | | | | | !spirv.Source = !{!0} | | | !0 = !{i32 3, i32 66048, !1} | | | ; 3 - OpenCL_C | | | ; 66048 = 0x10200 - OpenCL version 1.2 | | | ; !1 - optional file id. | | | !1 = !{!"/tmp/opencl/program.cl"} | +--------------------+---------------------------------------------------------+ | OpSourceExtension | .. code-block:: llvm | | | | | | !spirv.SourceExtension = !{!0, !1} | | | !0 = !{!"cl_khr_fp16"} | | | !1 = !{!"cl_khr_gl_sharing"} | +--------------------+---------------------------------------------------------+ | OpExtension | .. code-block:: llvm | | | | | | !spirv.Extension = !{!0} | | | !0 = !{!"SPV_KHR_expect_assume"} | +--------------------+---------------------------------------------------------+ | OpCapability | .. code-block:: llvm | | | | | | !spirv.Capability = !{!0} | | | !0 = !{i32 10} ; Float64 - program uses doubles | +--------------------+---------------------------------------------------------+ | OpExecutionMode | .. code-block:: llvm | | | | | | !spirv.ExecutionMode = !{!0} | | | !0 = !{void ()* @worker, i32 30, i32 262149} | | | ; Set execution mode with id 30 (VecTypeHint) and | | | ; literal `262149` operand. | +--------------------+---------------------------------------------------------+ | Generator's magic | .. code-block:: llvm | | number - word # 2 | | | in SPIR-V module | !spirv.Generator = !{!0} | | | !0 = !{i16 6, i16 123} | | | ; 6 - Generator Id, 123 - Generator Version | +--------------------+---------------------------------------------------------+ For example: .. code-block:: llvm !spirv.Source = !{!0} !spirv.SourceExtension = !{!2, !3} !spirv.Extension = !{!2} !spirv.Capability = !{!4} !spirv.MemoryModel = !{!5} !spirv.EntryPoint = !{!6 ,!7} !spirv.ExecutionMode = !{!8, !9} !spirv.Generator = !{!10 } ; 3 - OpenCL_C, 102000 - OpenCL version 1.2, !1 - optional file id. !0 = !{i32 3, i32 102000, !1} !1 = !{!"/tmp/opencl/program.cl"} !2 = !{!"cl_khr_fp16"} !3 = !{!"cl_khr_gl_sharing"} !4 = !{i32 10} ; Float64 - program uses doubles !5 = !{i32 1, i32 2} ; 1 - 32-bit addressing model, 2 - OpenCL memory model !6 = !{i32 6, TBD, !"kernel1", TBD} !7 = !{i32 6, TBD, !"kernel2", TBD} !8 = !{!6, i32 18, i32 16, i32 1, i32 1} ; local size hint <16, 1, 1> for 'kernel1' !9 = !{!7, i32 32} ; independent forward progress is required for 'kernel2' !10 = !{i16 6, i16 123} ; 6 - Generator Id, 123 - Generator Version Additional requirements for LLVM module ======================================= Target triple and datalayout string ----------------------------------- Target triple architecture must be ``spir`` (32-bit architecture) or ``spir64`` (64-bit architecture) and ``datalayout`` string must be aligned with OpenCL environment specification requirements for data type sizes and alignments (e.g. 3-element vector must have 4-element vector alignment). For example: .. code-block:: llvm target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir-unknown-unknown" Target triple architecture is translated to `addressing model operand `_ of `OpMemoryModel `_ SPIR-V instruction. - ``spir`` -> Physical32 - ``spir64`` -> Physical64 Calling convention ------------------ ``OpEntryPoint`` information is represented in LLVM IR in calling convention. A function with ``spir_kernel`` calling convention will be translated as an entry point of the SPIR-V module. Function metadata ----------------- Some kernel parameter information is stored in LLVM IR as a function metadata. For example: .. code-block:: llvm !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 **NOTE**: All metadata from the example above are optional. Access qualifiers are translated for image types, but they should be encoded in LLVM IR type name rather than function metadata. Function parameter, instruction and global variable decoration through metadata ------------------------------------------------------------------ Function parameters, instructions and global variables can be decorated using LLVM metadata through the metadata names ``spirv.ParameterDecorations`` and ``spirv.Decorations`` respectively. ``spirv.ParameterDecorations`` must be tied to the kernel function while ``spirv.Decorations`` is tied directly to the instruction or global variable. A "decoration-node" is a metadata node consisting of one or more operands. The first operand is an integer literal representing the SPIR-V decoration identifier. The other operands are either an integer or string literal representing the remaining extra operands of the corresponding SPIR-V decoration. A "decoration-list" is a metadata node consisting of references to zero or more decoration-nodes. ``spirv.Decorations`` must refer to a decoration-list while ``spirv.ParameterDecorations`` must refer to a metadata node that contains N references to decoration-lists, where N is the number of arguments of the function the metadata is tied to. ``spirv.Decorations`` applied on a global variable example: .. code-block:: llvm @v = global i32 0, !spirv.Decorations !1 ... !1 = !{!2, !3} ; decoration-list with two decoration nodes !2 = !{i32 22} ; decoration-node with no extra operands !3 = !{i32 41, !"v", i32 0} ; decoration-node with 2 extra operands decorates a global variable ``v`` with ``Constant`` and ``LinkageAttributes`` with extra operands ``"v"`` and ``Export`` in SPIR-V. ``spirv.Decorations`` applied on an instruction example: .. code-block:: llvm %idx = getelementptr inbounds i32, ptr addrspace(1) %b, i64 1, !spirv.Decorations !1 ... !1 = !{!2} !2 = !{i32 6442, i32 1, i32 2} ; {CacheControlLoadINTEL, CacheLevel=1, Cached} decorates getelementptr instruction with CacheControlLoadINTEL decoration with extra operands ``i32 1`` and ``i32 2``. ``spirv.ParameterDecorations`` example: .. code-block:: llvm define spir_kernel void @k(float %a, float %b) #0 !spirv.ParameterDecorations !1 ... !1 = !{!2, !3} ; metadata node with 2 decoration-lists !2 = !{} ; empty decoration-list !3 = !{!4} ; decoration-list with one decoration node !4 = !{i32 19} ; decoration-node with no extra operands decorates the argument ``b`` of ``k`` with ``Restrict`` in SPIR-V while not adding any decoration to argument ``a``. Member decoration through pointer annotations --------------------------------------------- Class members can be decorated using the ``llvm.ptr.annotation`` LLVM IR intrinsic. Member decorations specified in ``llvm.ptr.annotation`` must be in the second argument and must have the format ``{X}`` or ``{X:Y}`` where ``X`` is either one of the reserved names or an integer literal representing the SPIR-V decoration identifier and ``Y`` is 1 or more arguments separated by ",", where each argument must be either a word (including numbers) or a string enclosed by quotation marks. The ``llvm.ptr.annotation`` can contain any number decorations following this format. For example, both ``{5835:1,2,3}`` and ``{bank_bits:1,2,3}`` will result in the ``BankwidthINTEL`` decoration with literals 1, 2, and 3 attached to the annotated member. The translator accepts a number of reserved names that correspond to SPIR-V member decorations. +-----------------------+------------------+-----------------------------------+ | Decoration | Reserved Name | Note | +=======================+==================+===================================+ | RegisterINTEL | register | Additional arguments are ignored, | | | | but reverse translation will add | | | | a 1 argument, i.e. | | | | ``{register:1}``. | +-----------------------+------------------+-----------------------------------+ | MemoryINTEL | memory | | +-----------------------+------------------+-----------------------------------+ | NumbanksINTEL | numbanks | | +-----------------------+------------------+-----------------------------------+ | BankwidthINTEL | bankwidth | | +-----------------------+------------------+-----------------------------------+ | MaxPrivateCopiesINTEL | private_copies | | +-----------------------+------------------+-----------------------------------+ | SinglepumpINTEL | pump | Reserved name is shared with | | | | DoublepumpINTEL. SinglepumpINTEL | | | | will be selected if the argument | | | | is 2, i.e ``{pump:1}``. | +-----------------------+------------------+-----------------------------------+ | DoublepumpINTEL | pump | Reserved name is shared with | | | | SinglepumpINTEL. DoublepumpINTEL | | | | will be selected if the argument | | | | is 2, i.e ``{pump:2}``. | +-----------------------+------------------+-----------------------------------+ | MaxReplicatesINTEL | max_replicates | | +-----------------------+------------------+-----------------------------------+ | SimpleDualPortINTEL | simple_dual_port | Additional arguments are ignored, | | | | but reverse translation will add | | | | a 1 argument, i.e. | | | | ``{simple_dual_port:1}``. | +-----------------------+------------------+-----------------------------------+ | MergeINTEL | merge | Arguments of this are separated by| | | | ":" rather than ",", i.e. | | | | ``{merge:X:Y}``. | +-----------------------+------------------+-----------------------------------+ | BankBitsINTEL | bank_bits | | +-----------------------+------------------+-----------------------------------+ | ForcePow2DepthINTEL | force_pow2_depth | | +-----------------------+------------------+-----------------------------------+ None of the special requirements imposed from using the reserved names apply to using decoration identifiers directly. During reverse translation, the translator prioritizes reserved names over decoration identifiers, even if the member decoration was generated using the corresponding decoration identifier. For example, this means that translating ``{5825}`` to SPIR-V and back to LLVM IR will result in ``{register:1}`` being in the annotation string argument instead of the initial value. Debug information extension =========================== **TBD** SPIRV-LLVM-Translator-15.0.17/docs/SPIRVVersionsAndExtensionsHandling.rst000066400000000000000000000227121506646577400260050ustar00rootroot00000000000000======================================= SPIR-V versions and extensions handling ======================================= .. contents:: :local: Overview ======== This document describes how the translator makes decisions about using instructions from different version of the SPIR-V core and extension specifications. Being able to control the resulting SPIR-V version is important: the target consumer might be quite old, without support for new SPIR-V versions and there must be the possibility to control which version of the SPIR-V specification that will be used during translation. SPIR-V extensions is another thing which must be controllable. Extensions can update and re-define semantics and validation rules for existing SPIR-V entries and it is important to ensure that the translator is able to generate valid SPIR-V according to the core spec, without uses of any extensions if such SPIR-V was requested by user. For example, without such infrastructure it is impossible to disable use of ``SPV_KHR_no_integer_wrap_decoration`` - it will be always generated if corresponding LLVM IR counterparts are encountered in input module. It is worth mentioning that SPIR-V versions and extensions the handling of SPIR-V versions and extension is mostly important for the SPIR-V generation step. On the consumer side it is the responsibility of the consumer to analyze the incoming SPIR-V file and reject it if it contains something that is not supported by the consumer. However, translator simplifies this step for downstream users by checking version and extensions in SPIR-V module during ``readSpirv``/``readSpirvModule`` phases. SPIR-V Versions =============== SPIR-V Generation step ---------------------- By default translator selects version of generated SPIR-V file based on features used in this file. For example, if it contains ``dereferencable`` LLVM IR attribute, ``MaxByteOffset`` decoration will be generated and resulting SPIR-V version will be raised to 1.1. .. note:: There is no documentation about which exact features from newest SPIR-V spec versions will be used by the translator. If you are interested when or why a particular SPIR-V instruction is generated, please check this in the source code. Consider this as an implementation detail and if you disagree with something, you can always open an issue or submit pull request - contributions are welcome! There is one option to control the behavior of the translator with respect to the version of the SPIR-V file which is being generated/consumed. * ``--spirv-max-version=`` - instructs the translator to generate SPIR-V file corresponding to any spec version which is less than or equal to the specified one. Behavior of the translator is the same as by default with only one exception: resulting SPIR-V version cannot be raised higher than specified by this option. Allowed values are ``1.0``, ``1.1``, ``1.2``, ``1.3``, ``1.4``, ``1.5`` and ``1.6``. .. warning:: These two options are mutually exclusive and cannot be specified at the same time. If the translator encounters something that cannot be represented by set of allowed SPIR-V versions (which might contain only one version), it does one of the following things: * ignores LLVM IR entity in the input file. For example, ``dereferencable`` LLVM IR attribute can be ignored if it is not allowed to generate SPIR-V 1.1 and higher. * tries to represent LLVM IR entity with allowed instructions. For example, ``OpPtrEqual`` can be used if SPIR-V 1.4 is not allowed and can be emulated via ``OpConvertPtrToU`` + ``OpIEqual`` sequence. * emits error if LLVM IR entity cannot be ignored and cannot be emulated using available instructions. For example, if global constructors/destructors (represented by @llvm.global_ctors/@llvm.global_dtors) are present in a module then the translator should emit error if it cannot use SPIR-V 1.1 and higher where ``Initializer`` and ``Finalizer`` execution modes are described. SPIR-V Consumption step ----------------------- By default, translator consumes SPIR-V of any version which is supported. This behavior, however, can be controlled via the same switches described in the previous section. If one of the switches present and translator encountered SPIR-V file corresponding to a spec version which is not included into set of allowed SPIR-V versions, translator emits error. SPIR-V Extensions ================= SPIR-V Generation step ---------------------- By default, translator doesn't use any extensions. If it required to enable certain extension, the following command line option can be used: * ``--spirv-ext=`` - allows to control list of allowed/disallowed extensions. Valid value for this option is comma-separated list of extension names prefixed with ``+`` or ``-`` - plus means allow to use extension, minus means disallow to use extension. There is one more special value which can be used as extension name in this option: ``all`` - it affects all extension which are known to the translator. If ``--spirv-ext`` contains name of extension which is not know for the translator, it will emit error. Examples: * ``--spirv-ext=+SPV_KHR_no_integer_wrap_decoration,+SPV_INTEL_subgroups`` * ``--spirv-ext=+all,-SPV_INTEL_fpga_loop_controls`` .. warning:: Extension name cannot be allowed and disallowed at the same time: for inputs like ``--spirv-ext=+SPV_INTEL_subgroups,-SPV_INTEL_subgroups`` translator will emit error about invalid arguments. .. note:: Since by default during SPIR-V generation all extensions are disabled, this means that ``-all,`` is implicitly added at the beggining of the ``-spirv-ext`` value. If the translator encounters something that cannot be represented by set of allowed SPIR-V extensions (which might be empty), it does one of the following things: * ignores LLVM IR entity in the input file. For example, ``nsw``/``nuw`` LLVM IR attributes can be ignored if it is not allowed to generate SPIR-V 1.4 and ``SPV_KHR_no_integer_wrap_decoration`` extension is disallowed. * tries to represent LLVM IR entity with allowed instructions. Translator could translate calls to a new built-in functions defined by some extensions as usual call instructions without using special SPIR-V instructions. However, this could result in a strange SPIR-V and most likely will lead to errors during consumption. Having that, translator should emit errors if it encounters a call to a built-in function from an extension which must be represented as a special SPIR-V instruction from extension which wasn't allowed to be used. I.e. if translator knows that this certain LLVM IR entity belongs to an extension functionality and this extension is disallowed, it should emit error rather than emulating it. * emits error if LLVM IR entity cannot be ignored and cannot be emulated using available instructions. For example, new built-in types defined by ``cl_intel_device_side_avc_motion_estimation`` cannot be represented in SPIR-V if ``SPV_INTEL_device_side_avc_motion_estimation`` is disallowed. SPIR-V Consumption step ----------------------- By default, translator consumes SPIR-V regardless of list extensions which are used by the input file, i.e. all extensions are allowed by default during consumption step. .. note:: This is opposite to the generation step and this is done on purpose: to not broke workflows of existing users of the translator. .. note:: Since by default during SPIR-V consumption all extensions are enabled, this means that ``+all,`` is implicitly added at the beggining of the ``-spirv-ext`` value. This behavior, however, can be controlled via the same switches described in the previous section. If ``--spirv-ext`` switch presents, translator will emit error if it finds out that input SPIR-V file uses disallowed extension. .. note:: If the translator encounters unknown extension in the input SPIR-V file, it will emit error regardless of ``-spirv-ext`` option value. If one of the switches present and translator encountered SPIR-V file corresponding to a spec version which is not included into set of allowed SPIR-V versions, translator emits error. How to control translator behavior when using it as library =========================================================== When using translator as library it can be controlled via bunch of alternative APIs that have additional argument: ``TranslatorOpts`` object which encapsulates information about available SPIR-V versions and extensions. List of new APIs is: ``readSpirvModule``, ``writeSpirv`` and ``readSpirv``. .. note:: See ``LLVMSPIRVOpts.h`` for more details. How to get ``TranslatorOpts`` object ------------------------------------ 1. Default constructor. Equal to: ``--spirv-max-version=MaxKnownVersion --spirv-ext=-all`` .. note:: There is method ``TranslatorOpts::enableAllExtensions()`` that allows you to quickly enable all known extensions if it is needed. 2. Constructor which accepts all parameters Consumes both max SPIR-V version and optional map with extensions status (i.e. which one is allowed and which one is disallowed) Extensions status map ^^^^^^^^^^^^^^^^^^^^^ This map is defined as ``std::map`` and it is intended to show which extension is allowed to be used (``true`` as value) and which is not (``false`` as value). .. note:: If certain ``ExtensionID`` value is missed in the map, it automatically means that extension is not allowed to be used. This implies that by default, all extensions are disallowed. SPIRV-LLVM-Translator-15.0.17/include/000077500000000000000000000000001506646577400171365ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/include/LLVMSPIRVExtensions.inc000066400000000000000000000053251506646577400233140ustar00rootroot00000000000000#ifndef EXT #define EXT(X) #endif EXT(SPV_EXT_shader_atomic_float16_add) EXT(SPV_EXT_shader_atomic_float_add) EXT(SPV_EXT_shader_atomic_float_min_max) EXT(SPV_KHR_no_integer_wrap_decoration) EXT(SPV_KHR_float_controls) EXT(SPV_KHR_linkonce_odr) EXT(SPV_KHR_expect_assume) EXT(SPV_KHR_integer_dot_product) EXT(SPV_KHR_bit_instructions) EXT(SPV_KHR_uniform_group_instructions) EXT(SPV_KHR_subgroup_rotate) EXT(SPV_KHR_non_semantic_info) EXT(SPV_KHR_shader_clock) EXT(SPV_INTEL_subgroups) EXT(SPV_KHR_cooperative_matrix) EXT(SPV_INTEL_media_block_io) EXT(SPV_INTEL_device_side_avc_motion_estimation) EXT(SPV_INTEL_fpga_loop_controls) EXT(SPV_INTEL_fpga_memory_attributes) EXT(SPV_INTEL_fpga_memory_accesses) EXT(SPV_INTEL_unstructured_loop_controls) EXT(SPV_INTEL_fpga_reg) EXT(SPV_INTEL_blocking_pipes) EXT(SPV_INTEL_function_pointers) EXT(SPV_INTEL_kernel_attributes) EXT(SPV_INTEL_io_pipes) EXT(SPV_INTEL_inline_assembly) EXT(SPV_INTEL_arbitrary_precision_integers) EXT(SPV_INTEL_optimization_hints) EXT(SPV_INTEL_float_controls2) EXT(SPV_INTEL_vector_compute) EXT(SPV_INTEL_fast_composite) // TODO: to remove EXT(SPV_INTEL_usm_storage_classes) EXT(SPV_INTEL_fpga_buffer_location) EXT(SPV_INTEL_arbitrary_precision_fixed_point) EXT(SPV_INTEL_arbitrary_precision_floating_point) EXT(SPV_INTEL_variable_length_array) EXT(SPV_INTEL_fp_fast_math_mode) EXT(SPV_INTEL_fpga_cluster_attributes) EXT(SPV_INTEL_loop_fuse) EXT(SPV_INTEL_long_composites) EXT(SPV_INTEL_long_constant_composite) // TODO: rename to // SPV_INTEL_long_composites later EXT(SPV_EXT_optnone) EXT(SPV_INTEL_optnone) EXT(SPV_INTEL_fpga_dsp_control) EXT(SPV_INTEL_memory_access_aliasing) EXT(SPV_INTEL_fpga_invocation_pipelining_attributes) EXT(SPV_INTEL_token_type) EXT(SPV_INTEL_debug_module) EXT(SPV_INTEL_runtime_aligned) EXT(SPV_EXT_arithmetic_fence) EXT(SPV_INTEL_arithmetic_fence) EXT(SPV_INTEL_bfloat16_conversion) EXT(SPV_INTEL_joint_matrix) EXT(SPV_INTEL_hw_thread_queries) EXT(SPV_INTEL_global_variable_decorations) EXT(SPV_INTEL_global_variable_host_access) EXT(SPV_INTEL_global_variable_fpga_decorations) EXT(SPV_INTEL_complex_float_mul_div) EXT(SPV_INTEL_split_barrier) EXT(SPV_INTEL_masked_gather_scatter) EXT(SPV_INTEL_tensor_float32_conversion) // TODO: to remove old extension EXT(SPV_INTEL_tensor_float32_rounding) EXT(SPV_EXT_relaxed_printf_string_address_space) EXT(SPV_INTEL_fp_max_error) EXT(SPV_INTEL_cache_controls) EXT(SPV_INTEL_maximum_registers) EXT(SPV_INTEL_bindless_images) EXT(SPV_INTEL_2d_block_io) EXT(SPV_INTEL_subgroup_matrix_multiply_accumulate) EXT(SPV_KHR_bfloat16) EXT(SPV_INTEL_bfloat16_arithmetic) EXT(SPV_INTEL_ternary_bitwise_function) EXT(SPV_INTEL_int4) EXT(SPV_INTEL_function_variants) EXT(SPV_INTEL_shader_atomic_bfloat16) SPIRV-LLVM-Translator-15.0.17/include/LLVMSPIRVLib.h000066400000000000000000000251631506646577400213430ustar00rootroot00000000000000//===- LLVMSPIRVLib.h - Read and write SPIR-V binary ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file LLVMSPIRVLib.h /// /// This files declares functions and passes for translating between LLVM and /// SPIR-V. /// /// //===----------------------------------------------------------------------===// #ifndef SPIRV_H #define SPIRV_H #include "LLVMSPIRVOpts.h" #include #include namespace llvm { // Pass initialization functions need to be declared before inclusion of // PassSupport.h. class PassRegistry; void initializeLLVMToSPIRVLegacyPass(PassRegistry &); void initializeOCLToSPIRVLegacyPass(PassRegistry &); void initializeOCLTypeToSPIRVLegacyPass(PassRegistry &); void initializeSPIRVLowerBoolLegacyPass(PassRegistry &); void initializeSPIRVLowerConstExprLegacyPass(PassRegistry &); void initializeSPIRVLowerOCLBlocksLegacyPass(PassRegistry &); void initializeSPIRVLowerMemmoveLegacyPass(PassRegistry &); void initializeSPIRVLowerSaddIntrinsicsLegacyPass(PassRegistry &); void initializeSPIRVRegularizeLLVMLegacyPass(PassRegistry &); void initializeSPIRVToOCL12LegacyPass(PassRegistry &); void initializeSPIRVToOCL20LegacyPass(PassRegistry &); void initializePreprocessMetadataLegacyPass(PassRegistry &); void initializeSPIRVLowerBitCastToNonStandardTypeLegacyPass(PassRegistry &); class ModulePass; class FunctionPass; } // namespace llvm #include "llvm/IR/Module.h" namespace SPIRV { class SPIRVModule; /// \brief Check if a string contains SPIR-V binary. bool isSpirvBinary(const std::string &Img); #ifdef _SPIRV_SUPPORT_TEXT_FMT /// \brief Convert SPIR-V between binary and internal textual formats. /// This function is not thread safe and should not be used in multi-thread /// applications unless guarded by a critical section. /// \returns true if succeeds. bool convertSpirv(std::istream &IS, std::ostream &OS, std::string &ErrMsg, bool FromText, bool ToText); /// \brief Convert SPIR-V between binary and internal text formats. /// This function is not thread safe and should not be used in multi-thread /// applications unless guarded by a critical section. bool convertSpirv(std::string &Input, std::string &Out, std::string &ErrMsg, bool ToText); /// \brief Check if a string contains SPIR-V in internal text format. bool isSpirvText(std::string &Img); #endif /// \brief Load SPIR-V from istream as a SPIRVModule. /// \returns null on failure. std::unique_ptr readSpirvModule(std::istream &IS, std::string &ErrMsg); /// \brief Load SPIR-V from istream as a SPIRVModule. /// \returns null on failure. std::unique_ptr readSpirvModule(std::istream &IS, const SPIRV::TranslatorOpts &Opts, std::string &ErrMsg); /// This contains a pair of the pointer element type and an indirection /// parameter (to capture cases where an array of OpenCL types is used). typedef llvm::PointerIntPair PointerIndirectPair; struct SPIRVModuleReport { SPIRV::VersionNumber Version; uint32_t MemoryModel; uint32_t AddrModel; std::vector Extensions; std::vector ExtendedInstructionSets; std::vector Capabilities; }; /// \brief Partially load SPIR-V from the stream and decode only selected /// instructions that are needed to retrieve general information /// about the module. If this call fails, readSPIRVModule is /// expected to fail as well. /// \returns nullopt on failure. llvm::Optional getSpirvReport(std::istream &IS); llvm::Optional getSpirvReport(std::istream &IS, int &ErrCode); struct SPIRVModuleTextReport { std::string Version; std::string MemoryModel; std::string AddrModel; std::vector Extensions; std::vector ExtendedInstructionSets; std::vector Capabilities; }; /// \brief Create a human-readable form of the report returned by a call to /// getSpirvReport by decoding its binary fields. /// \returns String with the human-readable report. SPIRVModuleTextReport formatSpirvReport(const SPIRVModuleReport &Report); } // End namespace SPIRV namespace llvm { /// \brief Translate LLVM module to SPIR-V and write to ostream. /// \returns true if succeeds. bool writeSpirv(Module *M, std::ostream &OS, std::string &ErrMsg); /// \brief Load SPIR-V from istream and translate to LLVM module. /// \returns true if succeeds. bool readSpirv(LLVMContext &C, std::istream &IS, Module *&M, std::string &ErrMsg); /// \brief Translate LLVM module to SPIR-V and write to ostream. /// \returns true if succeeds. bool writeSpirv(Module *M, const SPIRV::TranslatorOpts &Opts, std::ostream &OS, std::string &ErrMsg); /// \brief Load SPIR-V from istream and translate to LLVM module. /// \returns true if succeeds. bool readSpirv(LLVMContext &C, const SPIRV::TranslatorOpts &Opts, std::istream &IS, Module *&M, std::string &ErrMsg); /// \brief Partially load SPIR-V from the stream and decode only instructions /// needed to get information about specialization constants. /// \returns true if succeeds. using SpecConstInfoTy = std::pair; bool getSpecConstInfo(std::istream &IS, std::vector &SpecConstInfo); /// \brief Convert a SPIRVModule into LLVM IR. /// \returns null on failure. std::unique_ptr convertSpirvToLLVM(LLVMContext &C, SPIRV::SPIRVModule &BM, std::string &ErrMsg); /// \brief Convert a SPIRVModule into LLVM IR using specified options /// \returns null on failure. std::unique_ptr convertSpirvToLLVM(LLVMContext &C, SPIRV::SPIRVModule &BM, const SPIRV::TranslatorOpts &Opts, std::string &ErrMsg); /// \brief Regularize LLVM module by removing entities not representable by /// SPIRV. bool regularizeLlvmForSpirv(Module *M, std::string &ErrMsg); bool regularizeLlvmForSpirv(Module *M, std::string &ErrMsg, const SPIRV::TranslatorOpts &Opts); /// \brief Mangle OpenCL builtin function function name. /// If any type in ArgTypes is a pointer type, the corresponding entry in /// ArgPointerTypes should contain the type should point to. If there are no /// pointer-typed arguments in ArgTypes, then ArgPointerTypes may be empty. void mangleOpenClBuiltin(const std::string &UnmangledName, ArrayRef ArgTypes, ArrayRef ArgPointerTypes, std::string &MangledName); /// Create a pass for translating LLVM to SPIR-V. ModulePass *createLLVMToSPIRVLegacy(SPIRV::SPIRVModule *); /// Create a pass for translating OCL C builtin functions to SPIR-V builtin /// functions. ModulePass *createOCLToSPIRVLegacy(); /// Create a pass for adapting OCL types for SPIRV. ModulePass *createOCLTypeToSPIRVLegacy(); /// Create a pass for lowering cast instructions of i1 type. ModulePass *createSPIRVLowerBoolLegacy(); /// Create a pass for lowering constant expressions to instructions. ModulePass *createSPIRVLowerConstExprLegacy(); /// Create a pass for removing function pointers related to OCL 2.0 blocks ModulePass *createSPIRVLowerOCLBlocksLegacy(); /// Create a pass for lowering llvm.memmove to llvm.memcpys with a temporary /// variable. ModulePass *createSPIRVLowerMemmoveLegacy(); /// Create a pass for lowering llvm.sadd.with.overflow ModulePass *createSPIRVLowerSaddIntrinsicsLegacy(); /// Create a pass for regularize LLVM module to be translated to SPIR-V. ModulePass *createSPIRVRegularizeLLVMLegacy(); /// Create a pass for translating SPIR-V Instructions to desired /// representation in LLVM IR (OpenCL built-ins, SPIR-V Friendly IR, etc.) ModulePass *createSPIRVBIsLoweringPass(Module &, SPIRV::BIsRepresentation); /// Create a pass for translating SPIR-V builtin functions to OCL 1.2 builtin /// functions. ModulePass *createSPIRVToOCL12Legacy(); /// Create a pass for translating SPIR-V builtin functions to OCL 2.0 builtin /// functions. ModulePass *createSPIRVToOCL20Legacy(); /// Create a pass for translating SPIR 1.2/2.0 metadata to SPIR-V friendly /// metadata. ModulePass *createPreprocessMetadataLegacy(); /// Create and return a pass that writes the module to the specified /// ostream. ModulePass *createSPIRVWriterPass(std::ostream &Str); /// Create and return a pass that writes the module to the specified /// ostream. ModulePass *createSPIRVWriterPass(std::ostream &Str, const SPIRV::TranslatorOpts &Opts); /// Create a pass for removing bitcast instructions to non-standard SPIR-V /// types FunctionPass *createSPIRVLowerBitCastToNonStandardTypeLegacy( const SPIRV::TranslatorOpts &Opts); } // namespace llvm #endif // SPIRV_H SPIRV-LLVM-Translator-15.0.17/include/LLVMSPIRVOpts.h000066400000000000000000000265461506646577400215700ustar00rootroot00000000000000//===- LLVMSPIRVOpts.h - Specify options for translation --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2019 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file LLVMSPIRVOpts.h /// /// This files declares helper classes to handle SPIR-V versions and extensions. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LLVMSPIRVOPTS_H #define SPIRV_LLVMSPIRVOPTS_H #include #include #include #include #include #include #include #include namespace llvm { class IntrinsicInst; } // namespace llvm namespace SPIRV { /// SPIR-V versions known to translator. enum class VersionNumber : uint32_t { // See section 2.3 of SPIR-V spec: Physical Layout of a SPIR_V Module and // Instruction SPIRV_1_0 = 0x00010000, SPIRV_1_1 = 0x00010100, SPIRV_1_2 = 0x00010200, SPIRV_1_3 = 0x00010300, SPIRV_1_4 = 0x00010400, SPIRV_1_5 = 0x00010500, SPIRV_1_6 = 0x00010600, MinimumVersion = SPIRV_1_0, MaximumVersion = SPIRV_1_6 }; inline std::string formatVersionNumber(uint32_t Version) { switch (Version) { case static_cast(VersionNumber::SPIRV_1_0): return "1.0"; case static_cast(VersionNumber::SPIRV_1_1): return "1.1"; case static_cast(VersionNumber::SPIRV_1_2): return "1.2"; case static_cast(VersionNumber::SPIRV_1_3): return "1.3"; case static_cast(VersionNumber::SPIRV_1_4): return "1.4"; } return "unknown"; } inline bool isSPIRVVersionKnown(uint32_t Ver) { return Ver >= static_cast(VersionNumber::MinimumVersion) && Ver <= static_cast(VersionNumber::MaximumVersion); } enum class ExtensionID : uint32_t { First, #define EXT(X) X, #include "LLVMSPIRVExtensions.inc" #undef EXT Last, }; enum class BIsRepresentation : uint32_t { OpenCL12, OpenCL20, SPIRVFriendlyIR }; enum class FPContractMode : uint32_t { On, Off, Fast }; enum class DebugInfoEIS : uint32_t { SPIRV_Debug, OpenCL_DebugInfo_100, NonSemantic_Shader_DebugInfo_100, NonSemantic_Shader_DebugInfo_200 }; enum class BuiltinFormat : uint32_t { Function, Global }; /// \brief Helper class to manage SPIR-V translation class TranslatorOpts { public: // Unset optional means not directly specified by user using ExtensionsStatusMap = std::map>; using ArgList = llvm::SmallVector; TranslatorOpts() = default; TranslatorOpts(VersionNumber Max, const ExtensionsStatusMap &Map = {}) : MaxVersion(Max), ExtStatusMap(Map) {} bool isAllowedToUseVersion(VersionNumber RequestedVersion) const { return RequestedVersion <= MaxVersion; } bool isAllowedToUseExtension(ExtensionID Extension) const { auto I = ExtStatusMap.find(Extension); if (ExtStatusMap.end() == I) return false; return I->second && *I->second; } void setAllowedToUseExtension(ExtensionID Extension, bool Allow = true) { // Only allow using the extension if it has not already been disabled auto I = ExtStatusMap.find(Extension); if (I == ExtStatusMap.end() || !I->second || (*I->second) == true) ExtStatusMap[Extension] = Allow; } VersionNumber getMaxVersion() const { return MaxVersion; } bool isGenArgNameMDEnabled() const { return GenKernelArgNameMD; } bool isSPIRVMemToRegEnabled() const { return SPIRVMemToReg; } void setMemToRegEnabled(bool Mem2Reg) { SPIRVMemToReg = Mem2Reg; } bool preserveAuxData() const { return PreserveAuxData; } void setPreserveAuxData(bool ArgValue) { PreserveAuxData = ArgValue; } void setGenKernelArgNameMDEnabled(bool ArgNameMD) { GenKernelArgNameMD = ArgNameMD; } void enableAllExtensions() { #define EXT(X) ExtStatusMap[ExtensionID::X] = true; #include "LLVMSPIRVExtensions.inc" #undef EXT } void enableGenArgNameMD() { GenKernelArgNameMD = true; } void setSpecConst(uint32_t SpecId, uint64_t SpecValue) { ExternalSpecialization[SpecId] = SpecValue; } bool getSpecializationConstant(uint32_t SpecId, uint64_t &Value) const { auto It = ExternalSpecialization.find(SpecId); if (It == ExternalSpecialization.end()) return false; Value = It->second; return true; } void setDesiredBIsRepresentation(BIsRepresentation Value) { DesiredRepresentationOfBIs = Value; } BIsRepresentation getDesiredBIsRepresentation() const { return DesiredRepresentationOfBIs; } void setFPContractMode(FPContractMode Mode) { FPCMode = Mode; } FPContractMode getFPContractMode() const { return FPCMode; } bool isUnknownIntrinsicAllowed(llvm::IntrinsicInst *II) const noexcept; bool isSPIRVAllowUnknownIntrinsicsEnabled() const noexcept; void setSPIRVAllowUnknownIntrinsics(ArgList IntrinsicPrefixList) noexcept; bool allowExtraDIExpressions() const noexcept { return AllowExtraDIExpressions; } void setAllowExtraDIExpressionsEnabled(bool Allow) noexcept { AllowExtraDIExpressions = Allow; } DebugInfoEIS getDebugInfoEIS() const { return DebugInfoVersion; } void setDebugInfoEIS(DebugInfoEIS EIS) { DebugInfoVersion = EIS; } bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept { return ReplaceLLVMFmulAddWithOpenCLMad; } void setReplaceLLVMFmulAddWithOpenCLMad(bool Value) noexcept { ReplaceLLVMFmulAddWithOpenCLMad = Value; } bool shouldPreserveOCLKernelArgTypeMetadataThroughString() const noexcept { return PreserveOCLKernelArgTypeMetadataThroughString; } void setPreserveOCLKernelArgTypeMetadataThroughString(bool Value) noexcept { PreserveOCLKernelArgTypeMetadataThroughString = Value; } bool shouldEmitFunctionPtrAddrSpace() const noexcept { return EmitFunctionPtrAddrSpace; } void setEmitFunctionPtrAddrSpace(bool Value) noexcept { EmitFunctionPtrAddrSpace = Value; } void setBuiltinFormat(BuiltinFormat Value) noexcept { SPIRVBuiltinFormat = Value; } BuiltinFormat getBuiltinFormat() const noexcept { return SPIRVBuiltinFormat; } void setFnVarCategory(uint32_t Category) noexcept { FnVarCategory = Category; } llvm::Optional getFnVarCategory() const noexcept { return FnVarCategory; } void setFnVarFamily(uint32_t Family) noexcept { FnVarFamily = Family; } llvm::Optional getFnVarFamily() const noexcept { return FnVarFamily; } void setFnVarArch(uint32_t Arch) noexcept { FnVarArch = Arch; } llvm::Optional getFnVarArch() const noexcept { return FnVarArch; } void setFnVarTarget(uint32_t Target) noexcept { FnVarTarget = Target; } llvm::Optional getFnVarTarget() const noexcept { return FnVarTarget; } void setFnVarFeatures(std::vector Features) noexcept { FnVarFeatures = Features; } std::vector getFnVarFeatures() const noexcept { return FnVarFeatures; } void setFnVarCapabilities(std::vector Capabilities) noexcept { FnVarCapabilities = Capabilities; } std::vector getFnVarCapabilities() const noexcept { return FnVarCapabilities; } void setFnVarSpecEnable(bool Val) noexcept { FnVarSpecEnable = Val; } bool getFnVarSpecEnable() const noexcept { return FnVarSpecEnable; } void setFnVarSpvOut(std::string Val) noexcept { FnVarSpvOut = Val; } std::string getFnVarSpvOut() const noexcept { return FnVarSpvOut; } // Check that options passed to --fnvar-xxx flags make sense. Return true on // success, false on failure. bool validateFnVarOpts() const; private: // Common translation options VersionNumber MaxVersion = VersionNumber::MaximumVersion; ExtensionsStatusMap ExtStatusMap; // SPIRVMemToReg option affects LLVM IR regularization phase bool SPIRVMemToReg = false; // SPIR-V to LLVM translation options bool GenKernelArgNameMD = false; std::unordered_map ExternalSpecialization; // Representation of built-ins, which should be used while translating from // SPIR-V to back to LLVM IR BIsRepresentation DesiredRepresentationOfBIs = BIsRepresentation::OpenCL12; // Controls floating point contraction. // // - FPContractMode::On allows to choose a mode according to // presence of fused LLVM intrinsics // // - FPContractMode::Off disables contratction for all entry points // // - FPContractMode::Fast allows *all* operations to be contracted // for all entry points FPContractMode FPCMode = FPContractMode::On; // Unknown LLVM intrinsics will be translated as external function calls in // SPIR-V llvm::Optional SPIRVAllowUnknownIntrinsics{}; // Enable support for extra DIExpression opcodes not listed in the SPIR-V // DebugInfo specification. bool AllowExtraDIExpressions = false; DebugInfoEIS DebugInfoVersion = DebugInfoEIS::OpenCL_DebugInfo_100; // Controls whether llvm.fmuladd.* should be replaced with mad from OpenCL // extended instruction set or with a simple fmul + fadd bool ReplaceLLVMFmulAddWithOpenCLMad = true; // Add a workaround to preserve OpenCL kernel_arg_type and // kernel_arg_type_qual metadata through OpString bool PreserveOCLKernelArgTypeMetadataThroughString = false; // Controls if CodeSectionINTEL can be emitted and consumed with a dedicated // address space bool EmitFunctionPtrAddrSpace = false; bool PreserveAuxData = false; llvm::Optional FnVarCategory; llvm::Optional FnVarFamily; llvm::Optional FnVarArch; llvm::Optional FnVarTarget; std::vector FnVarFeatures = {}; std::vector FnVarCapabilities = {}; std::string FnVarSpvOut = ""; bool FnVarSpecEnable = false; BuiltinFormat SPIRVBuiltinFormat = BuiltinFormat::Function; }; } // namespace SPIRV #endif // SPIRV_LLVMSPIRVOPTS_H SPIRV-LLVM-Translator-15.0.17/lib/000077500000000000000000000000001506646577400162615ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/000077500000000000000000000000001506646577400171645ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/CMakeLists.txt000066400000000000000000000030531506646577400217250ustar00rootroot00000000000000add_llvm_library(LLVMSPIRVLib LLVMSPIRVOpts.cpp LLVMToSPIRVDbgTran.cpp Mangler/FunctionDescriptor.cpp Mangler/Mangler.cpp Mangler/ManglingUtils.cpp Mangler/ParameterType.cpp OCLToSPIRV.cpp OCLTypeToSPIRV.cpp OCLUtil.cpp VectorComputeUtil.cpp SPIRVLowerBitCastToNonStandardType.cpp SPIRVLowerBool.cpp SPIRVLowerConstExpr.cpp SPIRVLowerMemmove.cpp SPIRVLowerOCLBlocks.cpp SPIRVLowerSaddIntrinsics.cpp SPIRVReader.cpp SPIRVRegularizeLLVM.cpp SPIRVToLLVMDbgTran.cpp SPIRVToOCL.cpp SPIRVToOCL12.cpp SPIRVToOCL20.cpp SPIRVTypeScavenger.cpp SPIRVUtil.cpp SPIRVWriter.cpp SPIRVWriterPass.cpp PreprocessMetadata.cpp libSPIRV/SPIRVBasicBlock.cpp libSPIRV/SPIRVDebug.cpp libSPIRV/SPIRVDecorate.cpp libSPIRV/SPIRVEntry.cpp libSPIRV/SPIRVFunction.cpp libSPIRV/SPIRVInstruction.cpp libSPIRV/SPIRVModule.cpp libSPIRV/SPIRVStream.cpp libSPIRV/SPIRVType.cpp libSPIRV/SPIRVValue.cpp libSPIRV/SPIRVFnVar.cpp LINK_COMPONENTS Analysis BitWriter CodeGen Core Demangle IRReader Linker Passes Support TransformUtils DEPENDS intrinsics_gen ) target_include_directories(LLVMSPIRVLib PRIVATE ${LLVM_INCLUDE_DIRS} ${LLVM_SPIRV_INCLUDE_DIRS} # TODO: Consider using SPIRV-Headers' as a header-only INTERFACE # instead. Right now this runs into exporting issues with # the LLVM in-tree builds. ${LLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/libSPIRV ${CMAKE_CURRENT_SOURCE_DIR}/Mangler ) SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/LLVMBuild.txt000066400000000000000000000013161506646577400214600ustar00rootroot00000000000000;===- ./lib/Target/SPIRV/Common/LLVMBuild.txt ------------------*- Conf -*--===; ; ; The LLVM Compiler Infrastructure ; ; This file is distributed under the University of Illinois Open Source ; License. See LICENSE.TXT for details. ; ;===------------------------------------------------------------------------===; ; ; This is an LLVMBuild description file for the components in this subdirectory. ; ; For more information on the LLVMBuild system, please see: ; ; http://llvm.org/docs/LLVMBuild.html ; ;===------------------------------------------------------------------------===; [component_0] type = Library name = SPIRVLib parent = Libraries required_libraries = Core Support Analysis IPO SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/LLVMSPIRVOpts.cpp000066400000000000000000000072641506646577400221450ustar00rootroot00000000000000//===- LLVMSPIRVOpts.cpp - Defines LLVM/SPIR-V options ----------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2021 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file provides definitions for LLVM/SPIR-V Translator's CLI /// functionality. /// //===----------------------------------------------------------------------===// #include "LLVMSPIRVOpts.h" #include #include #include #include #include using namespace llvm; using namespace SPIRV; bool TranslatorOpts::isUnknownIntrinsicAllowed( IntrinsicInst *II) const noexcept { if (!SPIRVAllowUnknownIntrinsics.hasValue()) return false; const auto &IntrinsicPrefixList = SPIRVAllowUnknownIntrinsics.getValue(); StringRef IntrinsicName = II->getCalledOperand()->getName(); for (const auto Prefix : IntrinsicPrefixList) { if (IntrinsicName.startswith(Prefix)) // Also true if `Prefix` is empty return true; } return false; } bool TranslatorOpts::isSPIRVAllowUnknownIntrinsicsEnabled() const noexcept { return SPIRVAllowUnknownIntrinsics.hasValue(); } void TranslatorOpts::setSPIRVAllowUnknownIntrinsics( TranslatorOpts::ArgList IntrinsicPrefixList) noexcept { SPIRVAllowUnknownIntrinsics = IntrinsicPrefixList; } bool TranslatorOpts::validateFnVarOpts() const { if (!getFnVarCategory().has_value() && (getFnVarFamily().has_value() || getFnVarArch().has_value())) { errs() << "FnVar: Device category must be specified if the family or " "architecture are specified."; return false; } if (!getFnVarFamily().has_value() && getFnVarArch().has_value()) { errs() << "FnVar: Device family must be specified if the architecture is " "specified."; return false; } if (!getFnVarTarget().has_value() && !getFnVarFeatures().empty()) { errs() << "Device target must be specified if the features are specified."; return false; } return true; } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/LLVMSaddWithOverflow.h000066400000000000000000000231021506646577400232610ustar00rootroot00000000000000//===- LLVMSaddWithOverflow.h - implementation of llvm.sadd.with.overflow -===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2020 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering of llvm.sadd.with.overflow.* into basic LLVM // operations. // //===----------------------------------------------------------------------===// // The IR below is slightly manually modified IR which was produced by Clang // from the C++ code below. The modifications include: // - adapting the return value, i.e. replacing `store` instructions for the c // and o arguments with `insertvalue` instructions. // - changing type of the `phi` instruction in the last basic block from `i8` // to `i1`. That also requires change of the argument of the `phi` instruction // and allowed to remove an unnecessary `sext` instruction. // // #include // #include // // const unsigned short i16_abs_pos_max = 0x7FFF; // 32767; // const unsigned short i16_abs_neg_max = 0x8000; // 32768; // // void llvm_sadd_with_overflow_i16(int16_t a, int16_t b, int16_t& c, bool& o) { // bool overflow = false; // bool both_pos = (a>=0 && b>=0); // bool both_neg = (a<0 && b<0); // if (both_pos || both_neg) { // // 32-bit integers are always supported in SPIR-V // uint32_t x = (uint32_t)abs(a) + (uint32_t)abs(b); // if (both_pos && x > i16_abs_pos_max || // both_neg && x > i16_abs_neg_max) { // overflow = true; // } // } // c = a + b; // o = overflow; // } // // const uint32_t i32_abs_pos_max = 0x7FFFFFFF; // 2147483647 // const uint32_t i32_abs_neg_max = 0x80000000; // 2147483648 // const int32_t i32_min = 0x80000000; // -2147483648 // // void llvm_sadd_with_overflow_i32(int32_t a, int32_t b, int32_t& c, bool& o) { // bool overflow = false; // bool both_pos = (a>=0 && b>=0); // bool both_neg = (a<0 && b<0); // // if a or b is the most negative number we can't get its absolute value, // // because it is out of range. // if (both_neg && (a == i32_min || b == i32_min)) // overflow = true; // else if (both_pos || both_neg) { // uint32_t x = (uint32_t)abs(a) + (uint32_t)abs(b); // if (both_pos && x > i32_abs_pos_max || // both_neg && x > 2147483648U) { // overflow = true; // } // } // c = a + b; // o = overflow; // } // // const uint64_t i64_abs_pos_max = 0x7fffffffffffffff; // 9223372036854775807 // const uint64_t i64_abs_neg_max = 0x8000000000000000; // 9223372036854775808 // const int64_t i64_min = 0x8000000000000000; // -9223372036854775808 // // void llvm_sadd_with_overflow_i64(int64_t a, int64_t b, int64_t& c, bool& o) { // bool overflow = false; // bool both_pos = (a>=0 && b>=0); // bool both_neg = (a<0 && b<0); // // if a or b is the most negative number we can't get its absolute value, // // because it is out of range. // if (both_neg && (a == i64_min || b == i64_min)) // overflow = true; // else if (both_pos || both_neg) { // uint64_t x = (uint64_t)abs(a) + (uint64_t)abs(b); // if (both_pos && x > i64_abs_pos_max || // both_neg && x > i64_abs_neg_max) { // overflow = true; // } // } // c = a + b; // o = overflow; // } // // const unsigned int abs_pos_max = 2147483647; // const unsigned int abs_neg_max = 2147483648; // // void llvm_sadd_with_overflow_i32(int a, int b, int& c, bool& o) { // bool overflow = false; // bool both_pos = (a>=0 && b>=0); // bool both_neg = (a<0 && b<0); // if (both_pos || both_neg) { // unsigned int x = (unsigned int)abs(a) + (unsigned int)abs(b); // if (both_pos && x > abs_pos_max || // both_neg && x > abs_neg_max) { // overflow = true; // } // } // c = a + b; // o = overflow; // } // Clang options: -emit-llvm -O2 -g0 -fno-discard-value-names static const char LLVMSaddWithOverflow[]{R"( define spir_func { i16, i1 } @llvm_sadd_with_overflow_i16(i16 %a, i16 %b) { entry: %conv = sext i16 %a to i32 %conv1 = sext i16 %b to i32 %0 = or i16 %b, %a %1 = icmp sgt i16 %0, -1 %2 = and i16 %b, %a %3 = icmp slt i16 %2, 0 %brmerge = or i1 %1, %3 br i1 %brmerge, label %if.then, label %if.end21 if.then: ; preds = %entry %4 = icmp slt i32 %conv, 0 %neg = sub nsw i32 0, %conv %5 = select i1 %4, i32 %neg, i32 %conv %6 = icmp slt i32 %conv1, 0 %neg39 = sub nsw i32 0, %conv1 %7 = select i1 %6, i32 %neg39, i32 %conv1 %add = add nuw nsw i32 %7, %5 %cmp15 = icmp ugt i32 %add, 32767 %or.cond = and i1 %1, %cmp15 %cmp19 = icmp ugt i32 %add, 32768 %or.cond28 = and i1 %3, %cmp19 %or.cond40 = or i1 %or.cond, %or.cond28 br label %if.end21 if.end21: ; preds = %if.then, %entry %overflow = phi i1 [ 0, %entry ], [ %or.cond40, %if.then ] %add24 = add i16 %b, %a %agg = insertvalue {i16, i1} undef, i16 %add24, 0 %res = insertvalue {i16, i1} %agg, i1 %overflow, 1 ret {i16, i1} %res } define spir_func { i32, i1 } @llvm_sadd_with_overflow_i32(i32 %a, i32 %b) { entry: %0 = or i32 %b, %a %1 = icmp sgt i32 %0, -1 %2 = and i32 %b, %a %3 = icmp slt i32 %2, 0 br i1 %3, label %land.lhs.true, label %if.else land.lhs.true: ; preds = %entry %cmp7 = icmp eq i32 %a, -2147483648 %cmp8 = icmp eq i32 %b, -2147483648 %or.cond = or i1 %cmp7, %cmp8 br i1 %or.cond, label %if.end23, label %if.then12 if.else: ; preds = %entry br i1 %1, label %if.then12, label %if.end23 if.then12: ; preds = %land.lhs.true, %if.else %4 = icmp slt i32 %a, 0 %neg = sub nsw i32 0, %a %5 = select i1 %4, i32 %neg, i32 %a %6 = icmp slt i32 %b, 0 %neg42 = sub nsw i32 0, %b %7 = select i1 %6, i32 %neg42, i32 %b %add = add nuw i32 %7, %5 %cmp16 = icmp slt i32 %add, 0 %or.cond27 = and i1 %1, %cmp16 %cmp20 = icmp ugt i32 %add, -2147483648 %or.cond28 = and i1 %3, %cmp20 %or.cond43 = or i1 %or.cond27, %or.cond28 br label %if.end23 if.end23: ; preds = %if.then12, %if.else, %land.lhs.true %overflow = phi i1 [ 1, %land.lhs.true ], [ 0, %if.else ], [ %or.cond43, %if.then12 ] %add24 = add nsw i32 %b, %a %agg = insertvalue {i32, i1} undef, i32 %add24, 0 %res = insertvalue {i32, i1} %agg, i1 %overflow, 1 ret {i32, i1} %res } define spir_func { i64, i1 } @llvm_sadd_with_overflow_i64(i64 %a, i64 %b) { entry: %0 = or i64 %b, %a %1 = icmp sgt i64 %0, -1 %2 = and i64 %b, %a %3 = icmp slt i64 %2, 0 br i1 %3, label %land.lhs.true, label %if.else land.lhs.true: ; preds = %entry %cmp7 = icmp eq i64 %a, -9223372036854775808 %cmp8 = icmp eq i64 %b, -9223372036854775808 %or.cond = or i1 %cmp7, %cmp8 br i1 %or.cond, label %if.end23, label %if.then12 if.else: ; preds = %entry br i1 %1, label %if.then12, label %if.end23 if.then12: ; preds = %land.lhs.true, %if.else %neg.i = sub nsw i64 0, %a %abscond.i = icmp slt i64 %a, 0 %abs.i = select i1 %abscond.i, i64 %neg.i, i64 %a %neg.i43 = sub nsw i64 0, %b %abscond.i44 = icmp slt i64 %b, 0 %abs.i45 = select i1 %abscond.i44, i64 %neg.i43, i64 %b %add = add nuw i64 %abs.i45, %abs.i %cmp16 = icmp slt i64 %add, 0 %or.cond27 = and i1 %1, %cmp16 %cmp20 = icmp ugt i64 %add, -9223372036854775808 %or.cond28 = and i1 %3, %cmp20 %or.cond42 = or i1 %or.cond27, %or.cond28 br label %if.end23 if.end23: ; preds = %if.then12, %if.else, %land.lhs.true %overflow = phi i1 [ 1, %land.lhs.true ], [ 0, %if.else ], [ %or.cond42, %if.then12 ] %add24 = add nsw i64 %b, %a %agg = insertvalue {i64, i1} undef, i64 %add24, 0 %res = insertvalue {i64, i1} %agg, i1 %overflow, 1 ret {i64, i1} %res } )"}; SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/LLVMToSPIRVDbgTran.cpp000066400000000000000000002037341506646577400230440ustar00rootroot00000000000000//===- LLVMToSPIRVDbgTran.cpp - Converts debug info to SPIR-V ---*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2018 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements translation of debug info from LLVM metadata to SPIR-V // //===----------------------------------------------------------------------===// #include "LLVMToSPIRVDbgTran.h" #include "SPIRV.debug.h" #include "SPIRVWriter.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" using namespace SPIRV; // Public interface /// This function is looking for debug information in the LLVM module /// and translates it to SPIRV void LLVMToSPIRVDbgTran::transDebugMetadata() { DIF.processModule(*M); if (DIF.compile_unit_count() == 0) return; if (isNonSemanticDebugInfo()) { if (!BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) BM->addExtension(SPIRV::ExtensionID::SPV_KHR_non_semantic_info); else BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_6); } for (DICompileUnit *CU : DIF.compile_units()) { transDbgEntry(CU); for (DIImportedEntity *IE : CU->getImportedEntities()) transDbgEntry(IE); } for (const DIType *T : DIF.types()) transDbgEntry(T); // When translating a debug lexical block, we expect the translation of its // parent scope (say it's a subprogram) already been created in MDMap. // Otherwise, we have to dive into the details of subprogram translation // first. During this process, we will try to resolve all retainedNodes // (aka, variables) owned by this subprogram. // And local variable's scope could be the original lexical block that we // haven't finish translating yet. In other words, the block hasn't been // inserted into MDMap cache yet. // So we try to invoke transDbgEntryImpl on the same lexical block again, // then we get a duplicated lexical block messing up the debug info. // // Scheduling the translation of subprograms ahead of scopes (lexical blocks) // solves this dependency cycle issue. for (const DISubprogram *F : DIF.subprograms()) transDbgEntry(F); for (const DIScope *S : DIF.scopes()) transDbgEntry(S); for (const DIGlobalVariableExpression *G : DIF.global_variables()) { transDbgEntry(G->getVariable()); } for (const DbgVariableIntrinsic *DDI : DbgDeclareIntrinsics) finalizeDebugDeclare(DDI); for (const DbgVariableIntrinsic *DVI : DbgValueIntrinsics) finalizeDebugValue(DVI); transLocationInfo(); } // llvm.dbg.declare intrinsic. SPIRVValue *LLVMToSPIRVDbgTran::createDebugDeclarePlaceholder( const DbgVariableIntrinsic *DbgDecl, SPIRVBasicBlock *BB) { DbgDeclareIntrinsics.push_back(DbgDecl); using namespace SPIRVDebug::Operand::DebugDeclare; SPIRVWordVec Ops(OperandCount, getDebugInfoNoneId()); SPIRVId ExtSetId = BM->getExtInstSetId(BM->getDebugInfoEIS()); return BM->addExtInst(getVoidTy(), ExtSetId, SPIRVDebug::Declare, Ops, BB); } void LLVMToSPIRVDbgTran::finalizeDebugDeclare( const DbgVariableIntrinsic *DbgDecl) { SPIRVValue *V = SPIRVWriter->getTranslatedValue(DbgDecl); assert(V && "llvm.dbg.declare intrinsic isn't mapped to a SPIRV instruction"); assert(V->isExtInst(BM->getDebugInfoEIS(), SPIRVDebug::Declare) && "llvm.dbg.declare intrinsic has been translated wrong!"); if (!V || !V->isExtInst(BM->getDebugInfoEIS(), SPIRVDebug::Declare)) return; SPIRVExtInst *DD = static_cast(V); SPIRVBasicBlock *BB = DD->getBasicBlock(); llvm::Value *Alloca = DbgDecl->getVariableLocationOp(0); using namespace SPIRVDebug::Operand::DebugDeclare; SPIRVWordVec Ops(OperandCount); Ops[DebugLocalVarIdx] = transDbgEntry(DbgDecl->getVariable())->getId(); Ops[VariableIdx] = Alloca ? SPIRVWriter->transValue(Alloca, BB)->getId() : getDebugInfoNoneId(); Ops[ExpressionIdx] = transDbgEntry(DbgDecl->getExpression())->getId(); DD->setArguments(Ops); } // llvm.dbg.value intrinsic. SPIRVValue *LLVMToSPIRVDbgTran::createDebugValuePlaceholder( const DbgVariableIntrinsic *DbgValue, SPIRVBasicBlock *BB) { if (!DbgValue->getVariableLocationOp(0)) return nullptr; // It is pointless without new value DbgValueIntrinsics.push_back(DbgValue); using namespace SPIRVDebug::Operand::DebugValue; SPIRVWordVec Ops(MinOperandCount, getDebugInfoNone()->getId()); SPIRVId ExtSetId = BM->getExtInstSetId(BM->getDebugInfoEIS()); return BM->addExtInst(getVoidTy(), ExtSetId, SPIRVDebug::Value, Ops, BB); } void LLVMToSPIRVDbgTran::finalizeDebugValue( const DbgVariableIntrinsic *DbgValue) { SPIRVValue *V = SPIRVWriter->getTranslatedValue(DbgValue); assert(V && "llvm.dbg.value intrinsic isn't mapped to a SPIRV instruction"); assert(V->isExtInst(BM->getDebugInfoEIS(), SPIRVDebug::Value) && "llvm.dbg.value intrinsic has been translated wrong!"); if (!V || !V->isExtInst(BM->getDebugInfoEIS(), SPIRVDebug::Value)) return; SPIRVExtInst *DV = static_cast(V); SPIRVBasicBlock *BB = DV->getBasicBlock(); Value *Val = DbgValue->getVariableLocationOp(0); DIExpression *Expr = DbgValue->getExpression(); if (DbgValue->getNumVariableLocationOps() > 1) { Val = UndefValue::get(Val->getType()); Expr = DIExpression::get(M->getContext(), {}); } using namespace SPIRVDebug::Operand::DebugValue; SPIRVWordVec Ops(MinOperandCount); Ops[DebugLocalVarIdx] = transDbgEntry(DbgValue->getVariable())->getId(); Ops[ValueIdx] = SPIRVWriter->transValue(Val, BB)->getId(); Ops[ExpressionIdx] = transDbgEntry(Expr)->getId(); DV->setArguments(Ops); } // Emitting DebugScope and OpLine instructions void LLVMToSPIRVDbgTran::transLocationInfo() { for (const Function &F : *M) { for (const BasicBlock &BB : F) { SPIRVValue *V = SPIRVWriter->getTranslatedValue(&BB); assert(V && V->isBasicBlock() && "Basic block is expected to be translated"); SPIRVBasicBlock *SBB = static_cast(V); MDNode *DbgScope = nullptr; MDNode *InlinedAt = nullptr; SPIRVString *File = nullptr; unsigned LineNo = 0; unsigned Col = 0; for (const Instruction &I : BB) { if (auto *II = dyn_cast(&I)) { if (II->getIntrinsicID() == Intrinsic::dbg_label) { // SPIR-V doesn't support llvm.dbg.label intrinsic translation continue; } if (II->getIntrinsicID() == Intrinsic::annotation || II->getIntrinsicID() == Intrinsic::var_annotation || II->getIntrinsicID() == Intrinsic::ptr_annotation) { // llvm call instruction for llvm .*annotation intrinsics // is translated into SPIR-V instruction only if it represents // call of __builtin_intel_fpga_reg() builtin. In other cases this // instruction is dropped. In these cases debug info for this call // should be skipped too. // TODO: Remove skipping of debug info when *.annotation call will // be handled in a better way during SPIR-V translation. V = SPIRVWriter->getTranslatedValue(&I); if (!V || V->getOpCode() != OpFPGARegINTEL) continue; } } V = SPIRVWriter->getTranslatedValue(&I); if (!V || isConstantOpCode(V->getOpCode())) continue; const DebugLoc &DL = I.getDebugLoc(); if (!DL.get()) { if (DbgScope || InlinedAt) { // Emit DebugNoScope DbgScope = nullptr; InlinedAt = nullptr; transDebugLoc(DL, SBB, static_cast(V)); } continue; } // Once scope or inlining has changed emit another DebugScope if (DL.getScope() != DbgScope || DL.getInlinedAt() != InlinedAt) { DbgScope = DL.getScope(); InlinedAt = DL.getInlinedAt(); transDebugLoc(DL, SBB, static_cast(V)); } // If any component of OpLine has changed emit another OpLine SPIRVString *DirAndFile = BM->getString(getFullPath(DL.get())); if (File != DirAndFile || LineNo != DL.getLine() || Col != DL.getCol()) { File = DirAndFile; LineNo = DL.getLine(); Col = DL.getCol(); // According to the spec, OpLine for an OpBranch/OpBranchConditional // must precede the merge instruction and not the branch instruction if (V->getOpCode() == OpBranch || V->getOpCode() == OpBranchConditional) { auto *VPrev = static_cast(V)->getPrevious(); if (VPrev && (VPrev->getOpCode() == OpLoopMerge || VPrev->getOpCode() == OpLoopControlINTEL)) { V = VPrev; } } if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) BM->addDebugLine(V, getVoidTy(), File ? File->getId() : getDebugInfoNoneId(), LineNo, LineNo, Col, Col + 1); else BM->addLine(V, File ? File->getId() : getDebugInfoNoneId(), LineNo, Col); } } // Instructions } // Basic Blocks } // Functions } // Translation of single debug entry SPIRVEntry *LLVMToSPIRVDbgTran::transDbgEntry(const MDNode *DIEntry) { // Caching auto It = MDMap.find(DIEntry); if (It != MDMap.end()) { assert(It->second && "Invalid SPIRVEntry is cached!"); return It->second; } SPIRVEntry *Res = transDbgEntryImpl(DIEntry); assert(Res && "Translation failure"); // We might end up having a recursive debug info generation like the // following: // translation of DIDerivedType (member) calls DICompositeType translation // as its parent scope; // translation of DICompositeType calls translation of its members // (DIDerivedType with member tag). // Here we make only the latest of these instructions be cached and hence // reused // FIXME: find a way to not create dead instruction if (MDMap[DIEntry]) return MDMap[DIEntry]; MDMap[DIEntry] = Res; return Res; } // Dispatcher implementation SPIRVEntry *LLVMToSPIRVDbgTran::transDbgEntryImpl(const MDNode *MDN) { if (!MDN) return BM->addDebugInfo(SPIRVDebug::DebugInfoNone, getVoidTy(), SPIRVWordVec()); if (const DINode *DIEntry = dyn_cast(MDN)) { switch (DIEntry->getTag()) { // Types case dwarf::DW_TAG_base_type: case dwarf::DW_TAG_unspecified_type: return transDbgBaseType(cast(DIEntry)); case dwarf::DW_TAG_reference_type: case dwarf::DW_TAG_rvalue_reference_type: case dwarf::DW_TAG_pointer_type: return transDbgPointerType(cast(DIEntry)); case dwarf::DW_TAG_array_type: return transDbgArrayType(cast(DIEntry)); case dwarf::DW_TAG_subrange_type: if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transDbgSubrangeType(cast(DIEntry)); else return getDebugInfoNone(); case dwarf::DW_TAG_string_type: { if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transDbgStringType(cast(DIEntry)); return getDebugInfoNone(); } case dwarf::DW_TAG_const_type: case dwarf::DW_TAG_restrict_type: case dwarf::DW_TAG_volatile_type: case dwarf::DW_TAG_atomic_type: return transDbgQualifiedType(cast(DIEntry)); case dwarf::DW_TAG_subroutine_type: return transDbgSubroutineType(cast(DIEntry)); case dwarf::DW_TAG_class_type: case dwarf::DW_TAG_structure_type: case dwarf::DW_TAG_union_type: return transDbgCompositeType(cast(DIEntry)); case dwarf::DW_TAG_member: return transDbgMemberType(cast(DIEntry)); case dwarf::DW_TAG_inheritance: return transDbgInheritance(cast(DIEntry)); case dwarf::DW_TAG_enumeration_type: return transDbgEnumType(cast(DIEntry)); case dwarf::DW_TAG_file_type: return transDbgFileType(cast(DIEntry)); case dwarf::DW_TAG_typedef: return transDbgTypeDef(cast(DIEntry)); case dwarf::DW_TAG_ptr_to_member_type: return transDbgPtrToMember(cast(DIEntry)); // Scope case dwarf::DW_TAG_namespace: case dwarf::DW_TAG_lexical_block: return transDbgScope(cast(DIEntry)); // Function case dwarf::DW_TAG_subprogram: return transDbgFunction(cast(DIEntry)); // Variables case dwarf::DW_TAG_variable: if (const DILocalVariable *LV = dyn_cast(DIEntry)) return transDbgLocalVariable(LV); if (const DIGlobalVariable *GV = dyn_cast(DIEntry)) return transDbgGlobalVariable(GV); llvm_unreachable("Unxpected debug info type for variable"); case dwarf::DW_TAG_formal_parameter: return transDbgLocalVariable(cast(DIEntry)); // Compilation unit case dwarf::DW_TAG_compile_unit: return transDbgCompileUnit(cast(DIEntry)); // Templates case dwarf::DW_TAG_template_type_parameter: case dwarf::DW_TAG_template_value_parameter: return transDbgTemplateParameter(cast(DIEntry)); case dwarf::DW_TAG_GNU_template_template_param: return transDbgTemplateTemplateParameter( cast(DIEntry)); case dwarf::DW_TAG_GNU_template_parameter_pack: return transDbgTemplateParameterPack( cast(DIEntry)); case dwarf::DW_TAG_imported_module: case dwarf::DW_TAG_imported_declaration: return transDbgImportedEntry(cast(DIEntry)); case dwarf::DW_TAG_module: { if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_debug_module) || BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transDbgModule(cast(DIEntry)); return getDebugInfoNone(); } default: return getDebugInfoNone(); } } if (const DIExpression *Expr = dyn_cast(MDN)) return transDbgExpression(Expr); if (const DILocation *Loc = dyn_cast(MDN)) { return transDbgInlinedAt(Loc); } llvm_unreachable("Not implemented debug info entry!"); } // Helper methods SPIRVType *LLVMToSPIRVDbgTran::getVoidTy() { if (!VoidT) { assert(M && "Pointer to LLVM Module is expected to be initialized!"); // Cache void type in a member. VoidT = SPIRVWriter->transType(Type::getVoidTy(M->getContext())); } return VoidT; } SPIRVType *LLVMToSPIRVDbgTran::getInt32Ty() { if (!Int32T) { assert(M && "Pointer to LLVM Module is expected to be initialized!"); // Cache int32 type in a member. Int32T = SPIRVWriter->transType(Type::getInt32Ty(M->getContext())); } return Int32T; } SPIRVEntry *LLVMToSPIRVDbgTran::getScope(DIScope *S) { if (S) return transDbgEntry(S); assert(!SPIRVCUMap.empty() && "Compile units are expected to be already translated"); return SPIRVCUMap.begin()->second; } SPIRVEntry *LLVMToSPIRVDbgTran::getGlobalVariable(const DIGlobalVariable *GV) { for (GlobalVariable &V : M->globals()) { SmallVector GVs; V.getDebugInfo(GVs); for (DIGlobalVariableExpression *GVE : GVs) { if (GVE->getVariable() == GV) return SPIRVWriter->transValue(&V, nullptr); } } return getDebugInfoNone(); } inline bool LLVMToSPIRVDbgTran::isNonSemanticDebugInfo() { return (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200); } void LLVMToSPIRVDbgTran::transformToConstant(std::vector &Ops, std::vector Idxs) { for (const auto Idx : Idxs) { SPIRVValue *Const = BM->addIntegerConstant( static_cast(getInt32Ty()), Ops[Idx]); Ops[Idx] = Const->getId(); } } SPIRVWord LLVMToSPIRVDbgTran::mapDebugFlags(DINode::DIFlags DFlags) { SPIRVWord Flags = 0; if ((DFlags & DINode::FlagAccessibility) == DINode::FlagPublic) Flags |= SPIRVDebug::FlagIsPublic; if ((DFlags & DINode::FlagAccessibility) == DINode::FlagProtected) Flags |= SPIRVDebug::FlagIsProtected; if ((DFlags & DINode::FlagAccessibility) == DINode::FlagPrivate) Flags |= SPIRVDebug::FlagIsPrivate; if (DFlags & DINode::FlagFwdDecl) Flags |= SPIRVDebug::FlagIsFwdDecl; if (DFlags & DINode::FlagArtificial) Flags |= SPIRVDebug::FlagIsArtificial; if (DFlags & DINode::FlagExplicit) Flags |= SPIRVDebug::FlagIsExplicit; if (DFlags & DINode::FlagPrototyped) Flags |= SPIRVDebug::FlagIsPrototyped; if (DFlags & DINode::FlagObjectPointer) Flags |= SPIRVDebug::FlagIsObjectPointer; if (DFlags & DINode::FlagStaticMember) Flags |= SPIRVDebug::FlagIsStaticMember; // inderect variable flag ? if (DFlags & DINode::FlagLValueReference) Flags |= SPIRVDebug::FlagIsLValueReference; if (DFlags & DINode::FlagRValueReference) Flags |= SPIRVDebug::FlagIsRValueReference; if (DFlags & DINode::FlagTypePassByValue) Flags |= SPIRVDebug::FlagTypePassByValue; if (DFlags & DINode::FlagTypePassByReference) Flags |= SPIRVDebug::FlagTypePassByReference; if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) if (DFlags & DINode::FlagBitField) Flags |= SPIRVDebug::FlagBitField; if (DFlags & DINode::FlagEnumClass) Flags |= SPIRVDebug::FlagIsEnumClass; return Flags; } SPIRVWord LLVMToSPIRVDbgTran::transDebugFlags(const DINode *DN) { SPIRVWord Flags = 0; if (const DIGlobalVariable *GV = dyn_cast(DN)) { if (GV->isLocalToUnit()) Flags |= SPIRVDebug::FlagIsLocal; if (GV->isDefinition()) Flags |= SPIRVDebug::FlagIsDefinition; } if (const DISubprogram *DS = dyn_cast(DN)) { if (DS->isLocalToUnit()) Flags |= SPIRVDebug::FlagIsLocal; if (DS->isOptimized()) Flags |= SPIRVDebug::FlagIsOptimized; if (DS->isDefinition()) Flags |= SPIRVDebug::FlagIsDefinition; Flags |= mapDebugFlags(DS->getFlags()); } if (DN->getTag() == dwarf::DW_TAG_reference_type) Flags |= SPIRVDebug::FlagIsLValueReference; if (DN->getTag() == dwarf::DW_TAG_rvalue_reference_type) Flags |= SPIRVDebug::FlagIsRValueReference; if (const DIType *DT = dyn_cast(DN)) Flags |= mapDebugFlags(DT->getFlags()); if (const DILocalVariable *DLocVar = dyn_cast(DN)) Flags |= mapDebugFlags(DLocVar->getFlags()); return Flags; } /// Clang doesn't emit access flags for members with default access specifier /// See clang/lib/CodeGen/CGDebugInfo.cpp: getAccessFlag() /// In SPIR-V we set the flags even for members with default access specifier SPIRVWord adjustAccessFlags(DIScope *Scope, SPIRVWord Flags) { if (Scope && (Flags & SPIRVDebug::FlagAccess) == 0) { unsigned Tag = Scope->getTag(); if (Tag == dwarf::DW_TAG_class_type) Flags |= SPIRVDebug::FlagIsPrivate; else if (Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type) Flags |= SPIRVDebug::FlagIsPublic; } return Flags; } // Fortran dynamic arrays can have following 'dataLocation', 'associated' // 'allocated' and 'rank' debug metadata. Such arrays are being mapped on // DebugTypeArrayDynamic from NonSemantic.Shader.200 debug spec inline bool isFortranArrayDynamic(const DICompositeType *AT) { return (AT->getRawDataLocation() || AT->getRawAssociated() || AT->getRawAllocated() || AT->getRawRank()); } /// The following methods (till the end of the file) implement translation of /// debug instrtuctions described in the spec. // Absent Debug Info SPIRVEntry *LLVMToSPIRVDbgTran::getDebugInfoNone() { if (!DebugInfoNone) { DebugInfoNone = transDbgEntry(nullptr); } return DebugInfoNone; } SPIRVId LLVMToSPIRVDbgTran::getDebugInfoNoneId() { return getDebugInfoNone()->getId(); } // Compilation unit SPIRVEntry *LLVMToSPIRVDbgTran::transDbgCompileUnit(const DICompileUnit *CU) { using namespace SPIRVDebug::Operand::CompilationUnit; SPIRVWordVec Ops(MinOperandCount); Ops[SPIRVDebugInfoVersionIdx] = SPIRVDebug::DebugInfoVersion; Ops[DWARFVersionIdx] = M->getDwarfVersion(); Ops[SourceIdx] = getSource(CU)->getId(); if (isNonSemanticDebugInfo()) generateBuildIdentifierAndStoragePath(CU); auto DwarfLang = static_cast(CU->getSourceLanguage()); Ops[LanguageIdx] = BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200 ? convertDWARFSourceLangToSPIRVNonSemanticDbgInfo(DwarfLang) : convertDWARFSourceLangToSPIRV(DwarfLang); if (isNonSemanticDebugInfo()) transformToConstant( Ops, {SPIRVDebugInfoVersionIdx, DWARFVersionIdx, LanguageIdx}); if (isNonSemanticDebugInfo()) { if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { Ops.push_back(BM->getString(CU->getProducer().str())->getId()); } } else { // TODO: Remove this workaround once we switch to NonSemantic.Shader.* debug // info by default BM->addModuleProcessed(SPIRVDebug::ProducerPrefix + CU->getProducer().str()); } // Cache CU in a member. SPIRVCUMap[CU] = static_cast( BM->addDebugInfo(SPIRVDebug::CompilationUnit, getVoidTy(), Ops)); return SPIRVCUMap[CU]; } // Types SPIRVEntry *LLVMToSPIRVDbgTran::transDbgBaseType(const DIBasicType *BT) { using namespace SPIRVDebug::Operand::TypeBasic; SPIRVWordVec Ops(OperandCountOCL); Ops[NameIdx] = BM->getString(BT->getName().str())->getId(); ConstantInt *Size = getUInt(M, BT->getSizeInBits()); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); auto Encoding = static_cast(BT->getEncoding()); SPIRVDebug::EncodingTag EncTag = SPIRVDebug::Unspecified; SPIRV::DbgEncodingMap::find(Encoding, &EncTag); // Unset encoding if it's complex and NonSemantic.Shader.DebugInfo.200 is not // enabled if (EncTag == SPIRVDebug::Complex && BM->getDebugInfoEIS() != SPIRVEIS_NonSemantic_Shader_DebugInfo_200) EncTag = SPIRVDebug::Unspecified; Ops[EncodingIdx] = EncTag; if (isNonSemanticDebugInfo()) { transformToConstant(Ops, {EncodingIdx}); // Flags value could not be generated by clang or by LLVM environment. Ops.push_back(getDebugInfoNoneId()); } return BM->addDebugInfo(SPIRVDebug::TypeBasic, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgPointerType(const DIDerivedType *PT) { using namespace SPIRVDebug::Operand::TypePointer; SPIRVWordVec Ops(OperandCount); SPIRVEntry *Base = transDbgEntry(PT->getBaseType()); Ops[BaseTypeIdx] = Base->getId(); Ops[StorageClassIdx] = ~0U; // all ones denote no address space Optional AS = PT->getDWARFAddressSpace(); if (AS.hasValue()) { SPIRAddressSpace SPIRAS = static_cast(AS.getValue()); Ops[StorageClassIdx] = SPIRSPIRVAddrSpaceMap::map(SPIRAS); } Ops[FlagsIdx] = transDebugFlags(PT); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {StorageClassIdx, FlagsIdx}); SPIRVEntry *Res = BM->addDebugInfo(SPIRVDebug::TypePointer, getVoidTy(), Ops); return Res; } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgQualifiedType(const DIDerivedType *QT) { using namespace SPIRVDebug::Operand::TypeQualifier; SPIRVWordVec Ops(OperandCount); SPIRVEntry *Base = transDbgEntry(QT->getBaseType()); Ops[BaseTypeIdx] = Base->getId(); Ops[QualifierIdx] = SPIRV::DbgTypeQulifierMap::map( static_cast(QT->getTag())); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {QualifierIdx}); return BM->addDebugInfo(SPIRVDebug::TypeQualifier, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgArrayType(const DICompositeType *AT) { if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { if (isFortranArrayDynamic(AT)) return transDbgArrayTypeDynamic(AT); return transDbgArrayTypeNonSemantic(AT); } return transDbgArrayTypeOpenCL(AT); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgArrayTypeOpenCL(const DICompositeType *AT) { using namespace SPIRVDebug::Operand::TypeArray; SPIRVWordVec Ops(MinOperandCount); Ops[BaseTypeIdx] = transDbgEntry(AT->getBaseType())->getId(); DINodeArray AR(AT->getElements()); // For N-dimensianal arrays AR.getNumElements() == N const unsigned N = AR.size(); Ops.resize(ComponentCountIdx + N); SPIRVWordVec LowerBounds(N); for (unsigned I = 0; I < N; ++I) { DISubrange *SR = cast(AR[I]); ConstantInt *Count = SR->getCount().get(); if (AT->isVector()) { assert(N == 1 && "Multidimensional vector is not expected!"); Ops[ComponentCountIdx] = static_cast(Count->getZExtValue()); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {ComponentCountIdx}); return BM->addDebugInfo(SPIRVDebug::TypeVector, getVoidTy(), Ops); } if (Count) { Ops[ComponentCountIdx + I] = SPIRVWriter->transValue(Count, nullptr)->getId(); } else { if (auto *UpperBound = dyn_cast(SR->getRawUpperBound())) Ops[ComponentCountIdx + I] = transDbgEntry(UpperBound)->getId(); else Ops[ComponentCountIdx + I] = getDebugInfoNoneId(); } if (auto *RawLB = SR->getRawLowerBound()) { if (auto *DIExprLB = dyn_cast(RawLB)) LowerBounds[I] = transDbgEntry(DIExprLB)->getId(); else { ConstantInt *ConstIntLB = SR->getLowerBound().get(); LowerBounds[I] = SPIRVWriter->transValue(ConstIntLB, nullptr)->getId(); } } else { LowerBounds[I] = getDebugInfoNoneId(); } } Ops.insert(Ops.end(), LowerBounds.begin(), LowerBounds.end()); return BM->addDebugInfo(SPIRVDebug::TypeArray, getVoidTy(), Ops); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgArrayTypeNonSemantic(const DICompositeType *AT) { using namespace SPIRVDebug::Operand::TypeArray; SPIRVWordVec Ops(MinOperandCount); Ops[BaseTypeIdx] = transDbgEntry(AT->getBaseType())->getId(); DINodeArray AR(AT->getElements()); // For N-dimensianal arrays AR.getNumElements() == N const unsigned N = AR.size(); Ops.resize(SubrangesIdx + N); for (unsigned I = 0; I < N; ++I) { DISubrange *SR = cast(AR[I]); ConstantInt *Count = SR->getCount().get(); if (AT->isVector()) { assert(N == 1 && "Multidimensional vector is not expected!"); Ops[ComponentCountIdx] = static_cast(Count->getZExtValue()); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {ComponentCountIdx}); return BM->addDebugInfo(SPIRVDebug::TypeVector, getVoidTy(), Ops); } Ops[SubrangesIdx + I] = transDbgEntry(SR)->getId(); } return BM->addDebugInfo(SPIRVDebug::TypeArray, getVoidTy(), Ops); } // The function is used to translate Fortran's dynamic arrays SPIRVEntry * LLVMToSPIRVDbgTran::transDbgArrayTypeDynamic(const DICompositeType *AT) { using namespace SPIRVDebug::Operand::TypeArrayDynamic; SPIRVWordVec Ops(MinOperandCount); Ops[BaseTypeIdx] = transDbgEntry(AT->getBaseType())->getId(); // DataLocation, Associated, Allocated and Rank can be either DIExpression // metadata or DIVariable auto TransOperand = [&](llvm::Metadata *DIMD) -> SPIRVWord { if (auto *DIExpr = dyn_cast_or_null(DIMD)) return transDbgExpression(DIExpr)->getId(); if (auto *DIVar = dyn_cast_or_null(DIMD)) { if (const DILocalVariable *LV = dyn_cast(DIVar)) return transDbgLocalVariable(LV)->getId(); if (const DIGlobalVariable *GV = dyn_cast(DIVar)) return transDbgGlobalVariable(GV)->getId(); } return getDebugInfoNoneId(); }; Ops[DataLocationIdx] = TransOperand(AT->getRawDataLocation()); Ops[AssociatedIdx] = TransOperand(AT->getRawAssociated()); Ops[AllocatedIdx] = TransOperand(AT->getRawAllocated()); Ops[RankIdx] = TransOperand(AT->getRawRank()); DINodeArray AR(AT->getElements()); // For N-dimensianal arrays AR.getNumElements() == N const unsigned N = AR.size(); Ops.resize(SubrangesIdx + N); for (unsigned I = 0; I < N; ++I) { DISubrange *SR = cast(AR[I]); Ops[SubrangesIdx + I] = transDbgEntry(SR)->getId(); } return BM->addDebugInfo(SPIRVDebug::TypeArrayDynamic, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgSubrangeType(const DISubrange *ST) { using namespace SPIRVDebug::Operand::TypeSubrange; SPIRVWordVec Ops(MinOperandCount); auto TransOperand = [&Ops, this, ST](int Idx) -> void { Metadata *RawNode = nullptr; switch (Idx) { case LowerBoundIdx: RawNode = ST->getRawLowerBound(); break; case UpperBoundIdx: RawNode = ST->getRawUpperBound(); break; case CountIdx: RawNode = ST->getRawCountNode(); break; } if (!RawNode) { Ops[Idx] = getDebugInfoNoneId(); return; } if (auto *Node = dyn_cast(RawNode)) { Ops[Idx] = transDbgEntry(Node)->getId(); } else { ConstantInt *IntNode = nullptr; switch (Idx) { case LowerBoundIdx: IntNode = ST->getLowerBound().get(); break; case UpperBoundIdx: IntNode = ST->getUpperBound().get(); break; case CountIdx: IntNode = ST->getCount().get(); break; case StrideIdx: IntNode = ST->getStride().get(); break; } Ops[Idx] = IntNode ? SPIRVWriter->transValue(IntNode, nullptr)->getId() : getDebugInfoNoneId(); } }; for (int Idx = 0; Idx < MinOperandCount; ++Idx) TransOperand(Idx); if (auto *RawNode = ST->getRawStride()) { Ops.resize(MaxOperandCount); if (auto *Node = dyn_cast(RawNode)) Ops[StrideIdx] = transDbgEntry(Node)->getId(); else Ops[StrideIdx] = SPIRVWriter->transValue(ST->getStride().get(), nullptr) ->getId(); } return BM->addDebugInfo(SPIRVDebug::TypeSubrange, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgStringType(const DIStringType *ST) { using namespace SPIRVDebug::Operand::TypeString; SPIRVWordVec Ops(MinOperandCount); Ops[NameIdx] = BM->getString(ST->getName().str())->getId(); Ops[BaseTypeIdx] = ST->getEncoding() ? getDebugInfoNoneId() /*TODO: replace with basetype*/ : getDebugInfoNoneId(); auto TransOperand = [&](llvm::Metadata *DIMD) -> SPIRVWord { if (auto *DIExpr = dyn_cast_or_null(DIMD)) return transDbgExpression(DIExpr)->getId(); if (auto *DIVar = dyn_cast_or_null(DIMD)) { if (const DILocalVariable *LV = dyn_cast(DIVar)) return transDbgLocalVariable(LV)->getId(); if (const DIGlobalVariable *GV = dyn_cast(DIVar)) return transDbgGlobalVariable(GV)->getId(); } return getDebugInfoNoneId(); }; Ops[DataLocationIdx] = TransOperand(ST->getRawStringLocationExp()); ConstantInt *Size = getUInt(M, ST->getSizeInBits()); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); if (auto *StrLengthExp = ST->getRawStringLengthExp()) { Ops[LengthAddrIdx] = TransOperand(StrLengthExp); } else if (auto *StrLengthVar = ST->getRawStringLength()) { Ops[LengthAddrIdx] = TransOperand(StrLengthVar); } else { Ops[LengthAddrIdx] = getDebugInfoNoneId(); } return BM->addDebugInfo(SPIRVDebug::TypeString, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgTypeDef(const DIDerivedType *DT) { using namespace SPIRVDebug::Operand::Typedef; SPIRVWordVec Ops(OperandCount); Ops[NameIdx] = BM->getString(DT->getName().str())->getId(); SPIRVEntry *BaseTy = transDbgEntry(DT->getBaseType()); assert(BaseTy && "Couldn't translate base type!"); Ops[BaseTypeIdx] = BaseTy->getId(); Ops[SourceIdx] = getSource(DT)->getId(); Ops[LineIdx] = 0; // This version of DIDerivedType has no line number Ops[ColumnIdx] = 0; // This version of DIDerivedType has no column number SPIRVEntry *Scope = getScope(DT->getScope()); assert(Scope && "Couldn't translate scope!"); Ops[ParentIdx] = Scope->getId(); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx}); return BM->addDebugInfo(SPIRVDebug::Typedef, getVoidTy(), Ops); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgSubroutineType(const DISubroutineType *FT) { using namespace SPIRVDebug::Operand::TypeFunction; SPIRVWordVec Ops(MinOperandCount); Ops[FlagsIdx] = transDebugFlags(FT); DITypeRefArray Types = FT->getTypeArray(); const size_t NumElements = Types.size(); if (NumElements) { Ops.resize(1 + NumElements); // First element of the TypeArray is the type of the return value, // followed by types of the function arguments' types. // The same order is preserved in SPIRV. for (unsigned I = 0; I < NumElements; ++I) Ops[ReturnTypeIdx + I] = transDbgEntry(Types[I])->getId(); } else { // void foo(); Ops[ReturnTypeIdx] = getVoidTy()->getId(); } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::TypeFunction, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgEnumType(const DICompositeType *ET) { using namespace SPIRVDebug::Operand::TypeEnum; SPIRVWordVec Ops(MinOperandCount); SPIRVEntry *UnderlyingType = getVoidTy(); if (DIType *DerivedFrom = ET->getBaseType()) UnderlyingType = transDbgEntry(DerivedFrom); ConstantInt *Size = getUInt(M, ET->getSizeInBits()); Ops[NameIdx] = BM->getString(ET->getName().str())->getId(); Ops[UnderlyingTypeIdx] = UnderlyingType->getId(); Ops[SourceIdx] = getSource(ET)->getId(); Ops[LineIdx] = ET->getLine(); Ops[ColumnIdx] = 0; // This version of DICompositeType has no column number Ops[ParentIdx] = getScope(ET->getScope())->getId(); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); Ops[FlagsIdx] = transDebugFlags(ET); DINodeArray Elements = ET->getElements(); size_t ElemCount = Elements.size(); for (unsigned I = 0; I < ElemCount; ++I) { DIEnumerator *E = cast(Elements[I]); ConstantInt *EnumValue = getInt(M, E->getValue().getSExtValue()); SPIRVValue *Val = SPIRVWriter->transValue(EnumValue, nullptr); assert(Val->getOpCode() == OpConstant && "LLVM constant must be translated to SPIRV constant"); Ops.push_back(Val->getId()); SPIRVString *Name = BM->getString(E->getName().str()); Ops.push_back(Name->getId()); } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::TypeEnum, getVoidTy(), Ops); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgCompositeType(const DICompositeType *CT) { using namespace SPIRVDebug::Operand::TypeComposite; SPIRVWordVec Ops(MinOperandCount); SPIRVForward *Tmp = BM->addForward(nullptr); MDMap.insert(std::make_pair(CT, Tmp)); auto Tag = static_cast(CT->getTag()); SPIRVId UniqId = getDebugInfoNoneId(); StringRef Identifier = CT->getIdentifier(); if (!Identifier.empty()) UniqId = BM->getString(Identifier.str())->getId(); ConstantInt *Size = getUInt(M, CT->getSizeInBits()); Ops[NameIdx] = BM->getString(CT->getName().str())->getId(); Ops[TagIdx] = SPIRV::DbgCompositeTypeMap::map(Tag); Ops[SourceIdx] = getSource(CT)->getId(); Ops[LineIdx] = CT->getLine(); Ops[ColumnIdx] = 0; // This version of DICompositeType has no column number Ops[ParentIdx] = getScope(CT->getScope())->getId(); Ops[LinkageNameIdx] = UniqId; Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); Ops[FlagsIdx] = transDebugFlags(CT); for (DINode *N : CT->getElements()) { Ops.push_back(transDbgEntry(N)->getId()); } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {TagIdx, LineIdx, ColumnIdx, FlagsIdx}); SPIRVEntry *Res = BM->addDebugInfo(SPIRVDebug::TypeComposite, getVoidTy(), Ops); // Translate template parameters. if (DITemplateParameterArray TP = CT->getTemplateParams()) { const unsigned int NumTParams = TP.size(); SPIRVWordVec Args(1 + NumTParams); Args[0] = Res->getId(); for (unsigned int I = 0; I < NumTParams; ++I) { Args[I + 1] = transDbgEntry(TP[I])->getId(); } Res = BM->addDebugInfo(SPIRVDebug::TypeTemplate, getVoidTy(), Args); } BM->replaceForward(Tmp, Res); MDMap[CT] = Res; return Res; } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgMemberType(const DIDerivedType *MT) { if (isNonSemanticDebugInfo()) return transDbgMemberTypeNonSemantic(MT); return transDbgMemberTypeOpenCL(MT); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgMemberTypeOpenCL(const DIDerivedType *MT) { using namespace SPIRVDebug::Operand::TypeMember::OpenCL; SPIRVWordVec Ops(MinOperandCount); Ops[NameIdx] = BM->getString(MT->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(MT->getBaseType())->getId(); Ops[SourceIdx] = getSource(MT)->getId(); Ops[LineIdx] = MT->getLine(); Ops[ColumnIdx] = 0; // This version of DIDerivedType has no column number Ops[ParentIdx] = transDbgEntry(MT->getScope())->getId(); ConstantInt *Offset = getUInt(M, MT->getOffsetInBits()); Ops[OffsetIdx] = SPIRVWriter->transValue(Offset, nullptr)->getId(); ConstantInt *Size = getUInt(M, MT->getSizeInBits()); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); Ops[FlagsIdx] = adjustAccessFlags(MT->getScope(), transDebugFlags(MT)); if (MT->isStaticMember()) { if (llvm::Constant *C = MT->getConstant()) { SPIRVValue *Val = SPIRVWriter->transValue(C, nullptr); assert(isConstantOpCode(Val->getOpCode()) && "LLVM constant must be translated to SPIRV constant"); Ops.push_back(Val->getId()); } } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::TypeMember, getVoidTy(), Ops); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgMemberTypeNonSemantic(const DIDerivedType *MT) { using namespace SPIRVDebug::Operand::TypeMember::NonSemantic; SPIRVWordVec Ops(MinOperandCount); Ops[NameIdx] = BM->getString(MT->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(MT->getBaseType())->getId(); Ops[SourceIdx] = getSource(MT)->getId(); Ops[LineIdx] = MT->getLine(); Ops[ColumnIdx] = 0; // This version of DIDerivedType has no column number ConstantInt *Offset = getUInt(M, MT->getOffsetInBits()); Ops[OffsetIdx] = SPIRVWriter->transValue(Offset, nullptr)->getId(); ConstantInt *Size = getUInt(M, MT->getSizeInBits()); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); Ops[FlagsIdx] = adjustAccessFlags(MT->getScope(), transDebugFlags(MT)); transDbgEntry(MT->getScope())->getId(); if (MT->isStaticMember()) { if (llvm::Constant *C = MT->getConstant()) { SPIRVValue *Val = SPIRVWriter->transValue(C, nullptr); assert(isConstantOpCode(Val->getOpCode()) && "LLVM constant must be translated to SPIRV constant"); Ops.push_back(Val->getId()); } } transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::TypeMember, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgInheritance(const DIDerivedType *DT) { using namespace SPIRVDebug::Operand::TypeInheritance; unsigned ParentIdx, OffsetIdx, SizeIdx, FlagsIdx, OperandCount; if (isNonSemanticDebugInfo()) { ParentIdx = NonSemantic::ParentIdx; OffsetIdx = NonSemantic::OffsetIdx; SizeIdx = NonSemantic::SizeIdx; FlagsIdx = NonSemantic::FlagsIdx; OperandCount = NonSemantic::OperandCount; } else { ParentIdx = OpenCL::ParentIdx; OffsetIdx = OpenCL::OffsetIdx; SizeIdx = OpenCL::SizeIdx; FlagsIdx = OpenCL::FlagsIdx; OperandCount = OpenCL::OperandCount; } SPIRVWordVec Ops(OperandCount); Ops[ParentIdx] = transDbgEntry(DT->getBaseType())->getId(); ConstantInt *OffsetInBits = getUInt(M, DT->getOffsetInBits()); Ops[OffsetIdx] = SPIRVWriter->transValue(OffsetInBits, nullptr)->getId(); ConstantInt *Size = getUInt(M, DT->getSizeInBits()); Ops[SizeIdx] = SPIRVWriter->transValue(Size, nullptr)->getId(); Ops[FlagsIdx] = transDebugFlags(DT); if (isNonSemanticDebugInfo()) { transformToConstant(Ops, {FlagsIdx}); } else { Ops[OpenCL::ChildIdx] = transDbgEntry(DT->getScope())->getId(); } return BM->addDebugInfo(SPIRVDebug::TypeInheritance, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgPtrToMember(const DIDerivedType *DT) { using namespace SPIRVDebug::Operand::TypePtrToMember; SPIRVWordVec Ops(OperandCount); Ops[MemberTypeIdx] = transDbgEntry(DT->getBaseType())->getId(); Ops[ParentIdx] = transDbgEntry(DT->getClassType())->getId(); return BM->addDebugInfo(SPIRVDebug::TypePtrToMember, getVoidTy(), Ops); } // Templates SPIRVEntry * LLVMToSPIRVDbgTran::transDbgTemplateParams(DITemplateParameterArray TPA, const SPIRVEntry *Target) { using namespace SPIRVDebug::Operand::TypeTemplate; SPIRVWordVec Ops(MinOperandCount); Ops[TargetIdx] = Target->getId(); for (DITemplateParameter *TP : TPA) { Ops.push_back(transDbgEntry(TP)->getId()); } return BM->addDebugInfo(SPIRVDebug::TypeTemplate, getVoidTy(), Ops); } SPIRVEntry * LLVMToSPIRVDbgTran::transDbgTemplateParameter(const DITemplateParameter *TP) { using namespace SPIRVDebug::Operand::TypeTemplateParameter; SPIRVWordVec Ops(OperandCount); Ops[NameIdx] = BM->getString(TP->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(TP->getType())->getId(); Ops[ValueIdx] = getDebugInfoNoneId(); if (TP->getTag() == dwarf::DW_TAG_template_value_parameter) { const DITemplateValueParameter *TVP = cast(TP); if (auto *TVVal = TVP->getValue()) { Constant *C = cast(TVVal)->getValue(); Ops[ValueIdx] = SPIRVWriter->transValue(C, nullptr)->getId(); } else { SPIRVType *TyPtr = SPIRVWriter->transType( PointerType::get(Type::getInt8Ty(M->getContext()), 0)); Ops[ValueIdx] = BM->addNullConstant(TyPtr)->getId(); } } Ops[SourceIdx] = getDebugInfoNoneId(); Ops[LineIdx] = 0; // This version of DITemplateParameter has no line number Ops[ColumnIdx] = 0; // This version of DITemplateParameter has no column info if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx}); return BM->addDebugInfo(SPIRVDebug::TypeTemplateParameter, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgTemplateTemplateParameter( const DITemplateValueParameter *TVP) { using namespace SPIRVDebug::Operand::TypeTemplateTemplateParameter; SPIRVWordVec Ops(OperandCount); assert(isa(TVP->getValue())); MDString *Val = cast(TVP->getValue()); Ops[NameIdx] = BM->getString(TVP->getName().str())->getId(); Ops[TemplateNameIdx] = BM->getString(Val->getString().str())->getId(); Ops[SourceIdx] = getDebugInfoNoneId(); Ops[LineIdx] = 0; // This version of DITemplateValueParameter has no line info Ops[ColumnIdx] = 0; // This version of DITemplateValueParameter has no column if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx}); return BM->addDebugInfo(SPIRVDebug::TypeTemplateTemplateParameter, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgTemplateParameterPack( const DITemplateValueParameter *TVP) { using namespace SPIRVDebug::Operand::TypeTemplateParameterPack; SPIRVWordVec Ops(MinOperandCount); assert(isa(TVP->getValue())); MDNode *Params = cast(TVP->getValue()); Ops[NameIdx] = BM->getString(TVP->getName().str())->getId(); Ops[SourceIdx] = getDebugInfoNoneId(); Ops[LineIdx] = 0; // This version of DITemplateValueParameter has no line info Ops[ColumnIdx] = 0; // This version of DITemplateValueParameter has no column for (const MDOperand &Op : Params->operands()) { SPIRVEntry *P = transDbgEntry(cast(Op.get())); Ops.push_back(P->getId()); } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx}); return BM->addDebugInfo(SPIRVDebug::TypeTemplateParameterPack, getVoidTy(), Ops); } // Global objects SPIRVEntry * LLVMToSPIRVDbgTran::transDbgGlobalVariable(const DIGlobalVariable *GV) { using namespace SPIRVDebug::Operand::GlobalVariable; SPIRVWordVec Ops(MinOperandCount); Ops[NameIdx] = BM->getString(GV->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(GV->getType())->getId(); Ops[SourceIdx] = getSource(GV)->getId(); Ops[LineIdx] = GV->getLine(); Ops[ColumnIdx] = 0; // This version of DIGlobalVariable has no column number // Parent scope DIScope *Context = GV->getScope(); SPIRVEntry *Parent = SPIRVCUMap.begin()->second; // Global variable may be declared in scope of a namespace or imported module, // it may also be a static variable declared in scope of a function. if (Context && (isa(Context) || isa(Context) || isa(Context))) Parent = transDbgEntry(Context); Ops[ParentIdx] = Parent->getId(); Ops[LinkageNameIdx] = BM->getString(GV->getLinkageName().str())->getId(); Ops[VariableIdx] = getGlobalVariable(GV)->getId(); Ops[FlagsIdx] = transDebugFlags(GV); // Check if GV is the definition of previously declared static member if (DIDerivedType *StaticMember = GV->getStaticDataMemberDeclaration()) Ops.push_back(transDbgEntry(StaticMember)->getId()); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::GlobalVariable, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgFunction(const DISubprogram *Func) { auto It = MDMap.find(Func); if (It != MDMap.end()) return static_cast(It->second); // As long as indexes of FunctionDeclaration operands match with Function using namespace SPIRVDebug::Operand::FunctionDeclaration; SPIRVWordVec Ops(OperandCount); Ops[NameIdx] = BM->getString(Func->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(Func->getType())->getId(); Ops[SourceIdx] = getSource(Func)->getId(); Ops[LineIdx] = Func->getLine(); Ops[ColumnIdx] = 0; // This version of DISubprogram has no column number auto Scope = Func->getScope(); if (Scope && !isa(Scope)) { Ops[ParentIdx] = getScope(Scope)->getId(); } else { if (auto *Unit = Func->getUnit()) Ops[ParentIdx] = SPIRVCUMap[Unit]->getId(); else // it might so happen, that DISubprogram is missing Unit parameter Ops[ParentIdx] = SPIRVCUMap.begin()->second->getId(); } Ops[LinkageNameIdx] = BM->getString(Func->getLinkageName().str())->getId(); Ops[FlagsIdx] = adjustAccessFlags(Scope, transDebugFlags(Func)); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); SPIRVEntry *DebugFunc = nullptr; SPIRVValue *FuncDef = nullptr; bool IsEntryPointKernel = false; if (!Func->isDefinition()) { DebugFunc = BM->addDebugInfo(SPIRVDebug::FunctionDeclaration, getVoidTy(), Ops); } else { // Here we add operands specific function definition using namespace SPIRVDebug::Operand::Function; Ops.resize(MinOperandCount); Ops[ScopeLineIdx] = Func->getScopeLine(); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {ScopeLineIdx}); Ops[FunctionIdIdx] = getDebugInfoNoneId(); for (const llvm::Function &F : M->functions()) { if (Func->describes(&F)) { // Function definition of spir_kernel can have no "spir_kernel" calling // convention because SPIRVRegularizeLLVMBase::addKernelEntryPoint pass // could have turned it to spir_func. The "true" entry point is a // wrapper kernel function, which can be found further in the module. if (FuncDef) { if (F.getCallingConv() == CallingConv::SPIR_KERNEL) { IsEntryPointKernel = true; break; } continue; } SPIRVValue *SPIRVFunc = SPIRVWriter->getTranslatedValue(&F); assert(SPIRVFunc && "All function must be already translated"); Ops[FunctionIdIdx] = SPIRVFunc->getId(); FuncDef = SPIRVFunc; if (!isNonSemanticDebugInfo()) break; // Most likely unreachable because of Regularise LLVM pass if (F.getCallingConv() == CallingConv::SPIR_KERNEL) { IsEntryPointKernel = true; break; } } } // For NonSemantic.Shader.DebugInfo we store Function Id index as a // separate DebugFunctionDefinition instruction. if (isNonSemanticDebugInfo()) Ops.pop_back(); if (DISubprogram *FuncDecl = Func->getDeclaration()) Ops.push_back(transDbgEntry(FuncDecl)->getId()); else { Ops.push_back(getDebugInfoNoneId()); if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { // Translate targetFuncName mostly for Fortran trampoline function if it // is the case StringRef TargetFunc = Func->getTargetFuncName(); if (!TargetFunc.empty()) Ops.push_back(BM->getString(TargetFunc.str())->getId()); } } DebugFunc = BM->addDebugInfo(SPIRVDebug::Function, getVoidTy(), Ops); MDMap.insert(std::make_pair(Func, DebugFunc)); // Functions local variable might be not refered to anywhere else, except // here. // Just translate them. for (const DINode *Var : Func->getRetainedNodes()) transDbgEntry(Var); } // If the function has template parameters the function *is* a template. if (DITemplateParameterArray TPA = Func->getTemplateParams()) { DebugFunc = transDbgTemplateParams(TPA, DebugFunc); } if (isNonSemanticDebugInfo() && (Func->isMainSubprogram() || IsEntryPointKernel)) [[maybe_unused]] SPIRVEntry *Inst = transDbgEntryPoint(Func, DebugFunc); if (isNonSemanticDebugInfo() && FuncDef) [[maybe_unused]] SPIRVEntry *Inst = transDbgFuncDefinition(FuncDef, DebugFunc); return DebugFunc; } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgFuncDefinition(SPIRVValue *FuncDef, SPIRVEntry *DbgFunc) { using namespace SPIRVDebug::Operand::FunctionDefinition; SPIRVWordVec Ops(OperandCount); Ops[FunctionIdx] = DbgFunc->getId(); Ops[DefinitionIdx] = FuncDef->getId(); SPIRVFunction *F = static_cast(FuncDef); SPIRVBasicBlock *BB = F->getNumBasicBlock() ? F->getBasicBlock(0) : nullptr; SPIRVId ExtSetId = BM->getExtInstSetId(BM->getDebugInfoEIS()); return BM->addExtInst(getVoidTy(), ExtSetId, SPIRVDebug::FunctionDefinition, Ops, BB, BB->getInst(0)); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgEntryPoint(const DISubprogram *Func, SPIRVEntry *DbgFunc) { using namespace SPIRVDebug::Operand::EntryPoint; SPIRVWordVec Ops(OperandCount); Ops[EntryPointIdx] = DbgFunc->getId(); DICompileUnit *CU = Func->getUnit(); if (!CU) { Ops[CompilationUnitIdx] = SPIRVCUMap.begin()->second->getId(); SPIRVWord EmptyStrIdx = BM->getString("")->getId(); Ops[CompilerSignatureIdx] = EmptyStrIdx; Ops[CommandLineArgsIdx] = EmptyStrIdx; return BM->addDebugInfo(SPIRVDebug::EntryPoint, getVoidTy(), Ops); } StringRef Producer = CU->getProducer(); StringRef Flags = CU->getFlags(); SPIRVEntry *CUVal = SPIRVCUMap[CU] ? SPIRVCUMap[CU] : getDebugInfoNone(); Ops[CompilationUnitIdx] = CUVal->getId(); Ops[CompilerSignatureIdx] = BM->getString(Producer.str())->getId(); Ops[CommandLineArgsIdx] = BM->getString(Flags.str())->getId(); return BM->addDebugInfo(SPIRVDebug::EntryPoint, getVoidTy(), Ops); } // Location information SPIRVEntry *LLVMToSPIRVDbgTran::transDbgScope(const DIScope *S) { if (const DILexicalBlockFile *LBF = dyn_cast(S)) { using namespace SPIRVDebug::Operand::LexicalBlockDiscriminator; SPIRVWordVec Ops(OperandCount); Ops[SourceIdx] = getSource(S)->getId(); Ops[DiscriminatorIdx] = LBF->getDiscriminator(); Ops[ParentIdx] = getScope(S->getScope())->getId(); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {DiscriminatorIdx}); return BM->addDebugInfo(SPIRVDebug::LexicalBlockDiscriminator, getVoidTy(), Ops); } using namespace SPIRVDebug::Operand::LexicalBlock; SPIRVWordVec Ops(MinOperandCount); Ops[SourceIdx] = getSource(S)->getId(); Ops[ParentIdx] = getScope(S->getScope())->getId(); if (const DILexicalBlock *LB = dyn_cast(S)) { Ops[LineIdx] = LB->getLine(); Ops[ColumnIdx] = LB->getColumn(); } else if (const DINamespace *NS = dyn_cast(S)) { Ops[LineIdx] = 0; // This version of DINamespace has no line number Ops[ColumnIdx] = 0; // This version of DINamespace has no column number Ops.push_back(BM->getString(NS->getName().str())->getId()); if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { SPIRVValue *ExpConst = BM->addConstant( SPIRVWriter->transType(Type::getInt1Ty(M->getContext())), NS->getExportSymbols() /*Is inlined namespace*/); Ops.push_back(ExpConst->getId()); } } if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx}); return BM->addDebugInfo(SPIRVDebug::LexicalBlock, getVoidTy(), Ops); } // Generating DebugScope and DebugNoScope instructions. They can interleave with // core instructions. SPIRVEntry *LLVMToSPIRVDbgTran::transDebugLoc(const DebugLoc &Loc, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore) { SPIRVId ExtSetId = BM->getExtInstSetId(BM->getDebugInfoEIS()); if (!Loc.get()) return BM->addExtInst(getVoidTy(), ExtSetId, SPIRVDebug::NoScope, std::vector(), BB, InsertBefore); using namespace SPIRVDebug::Operand::Scope; SPIRVWordVec Ops(MinOperandCount); Ops[ScopeIdx] = getScope(static_cast(Loc.getScope()))->getId(); if (DILocation *IA = Loc.getInlinedAt()) Ops.push_back(transDbgEntry(IA)->getId()); return BM->addExtInst(getVoidTy(), ExtSetId, SPIRVDebug::Scope, Ops, BB, InsertBefore); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgInlinedAt(const DILocation *Loc) { // There is a Column operand in NonSemantic.Shader.200 spec if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transDbgInlinedAtNonSemanticShader200(Loc); using namespace SPIRVDebug::Operand::InlinedAt::OpenCL; SPIRVWordVec Ops(MinOperandCount); Ops[LineIdx] = Loc->getLine(); Ops[ScopeIdx] = getScope(Loc->getScope())->getId(); if (DILocation *IA = Loc->getInlinedAt()) Ops.push_back(transDbgEntry(IA)->getId()); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx}); return BM->addDebugInfo(SPIRVDebug::InlinedAt, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgInlinedAtNonSemanticShader200( const DILocation *Loc) { using namespace SPIRVDebug::Operand::InlinedAt::NonSemantic; SPIRVWordVec Ops(MinOperandCount); Ops[LineIdx] = Loc->getLine(); Ops[ColumnIdx] = Loc->getColumn(); transformToConstant(Ops, {LineIdx, ColumnIdx}); Ops[ScopeIdx] = getScope(Loc->getScope())->getId(); if (DILocation *IA = Loc->getInlinedAt()) Ops.push_back(transDbgEntry(IA)->getId()); return BM->addDebugInfo(SPIRVDebug::InlinedAt, getVoidTy(), Ops); } template SPIRVExtInst *LLVMToSPIRVDbgTran::getSource(const T *DIEntry) { const std::string FileName = getFullPath(DIEntry); auto It = FileMap.find(FileName); if (It != FileMap.end()) return It->second; using namespace SPIRVDebug::Operand::Source; SPIRVWordVec Ops(MinOperandCount); Ops[FileIdx] = BM->getString(FileName)->getId(); DIFile *F = DIEntry ? DIEntry->getFile() : nullptr; if (F && F->getRawChecksum()) { auto CheckSum = F->getChecksum().getValue(); if (!isNonSemanticDebugInfo()) Ops.push_back(BM->getString("//__" + CheckSum.getKindAsString().str() + ":" + CheckSum.Value.str()) ->getId()); else if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { const SPIRVDebug::FileChecksumKind ChecksumKind = SPIRV::DbgChecksumKindMap::map(CheckSum.Kind); Ops.push_back( BM->addIntegerConstant(static_cast(getInt32Ty()), ChecksumKind) ->getId()); Ops.push_back(BM->getString(CheckSum.Value.str())->getId()); } } if (F && F->getRawSource() && isNonSemanticDebugInfo()) { std::string Str = F->getSource().getValue().str(); constexpr size_t MaxNumWords = MaxWordCount - 2 /*Fixed WC for SPIRVString*/; constexpr size_t MaxStrSize = MaxNumWords * 4 - 1; const size_t NumWords = getSizeInWords(Str); if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200 && Ops.size() == MinOperandCount) { Ops.push_back(getDebugInfoNoneId()); Ops.push_back(getDebugInfoNoneId()); } Ops.push_back(BM->getString(Str.substr(0, MaxStrSize))->getId()); SPIRVExtInst *Source = static_cast( BM->addDebugInfo(SPIRVDebug::Source, getVoidTy(), Ops)); FileMap[FileName] = Source; Str.erase(0, MaxStrSize); // No need to generate source continued instructions if (NumWords < MaxNumWords) return Source; const uint64_t NumOfContinuedInstructions = NumWords / MaxNumWords - 1 + (NumWords % MaxNumWords ? 1 : 0); for (uint64_t J = 0; J < NumOfContinuedInstructions; J++) { const SPIRVWord Op = BM->getString(Str.substr(0, MaxStrSize))->getId(); BM->addDebugInfo(SPIRVDebug::SourceContinued, getVoidTy(), {Op}); Str.erase(0, MaxStrSize); } return Source; } SPIRVExtInst *Source = static_cast( BM->addDebugInfo(SPIRVDebug::Source, getVoidTy(), Ops)); FileMap[FileName] = Source; return Source; } void LLVMToSPIRVDbgTran::generateBuildIdentifierAndStoragePath( const DICompileUnit *DIEntry) { // get information from LLVM IR auto BuildIdentifier = DIEntry->getDWOId(); const std::string BuildIdentifierString = std::to_string(BuildIdentifier); const std::string StoragePath = DIEntry->getSplitDebugFilename().str(); using namespace SPIRVDebug::Operand; if (BuildIdentifierInsn || StoragePathInsn) { #ifndef NDEBUG assert(BuildIdentifierInsn && StoragePathInsn && "BuildIdentifier and StoragePath instructions must both be created"); auto PreviousBuildIdentifierString = BM->get( BuildIdentifierInsn ->getArguments()[BuildIdentifier::IdentifierIdx]) ->getStr(); assert(PreviousBuildIdentifierString == BuildIdentifierString && "New BuildIdentifier should match previous BuildIdentifier"); auto PreviousStoragePath = BM->get( StoragePathInsn->getArguments()[StoragePath::PathIdx]) ->getStr(); assert(PreviousStoragePath == StoragePath && "New StoragePath should match previous StoragePath"); #endif return; } // generate BuildIdentifier inst SPIRVWordVec BuildIdentifierOps(BuildIdentifier::OperandCount); BuildIdentifierOps[BuildIdentifier::IdentifierIdx] = BM->getString(BuildIdentifierString)->getId(); BuildIdentifierOps[BuildIdentifier::FlagsIdx] = BM->getLiteralAsConstant(1)->getId(); // Placeholder value for now BuildIdentifierInsn = static_cast(BM->addDebugInfo( SPIRVDebug::BuildIdentifier, getVoidTy(), BuildIdentifierOps)); // generate StoragePath inst SPIRVWordVec StoragePathOps(StoragePath::OperandCount); StoragePathOps[StoragePath::PathIdx] = BM->getString(StoragePath)->getId(); StoragePathInsn = static_cast( BM->addDebugInfo(SPIRVDebug::StoragePath, getVoidTy(), StoragePathOps)); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgFileType(const DIFile *F) { return BM->getString(getFullPath(F)); } // Local variables SPIRVEntry * LLVMToSPIRVDbgTran::transDbgLocalVariable(const DILocalVariable *Var) { using namespace SPIRVDebug::Operand::LocalVariable; SPIRVWordVec Ops(MinOperandCount); Ops[NameIdx] = BM->getString(Var->getName().str())->getId(); Ops[TypeIdx] = transDbgEntry(Var->getType())->getId(); Ops[SourceIdx] = getSource(Var->getFile())->getId(); Ops[LineIdx] = Var->getLine(); Ops[ColumnIdx] = 0; // This version of DILocalVariable has no column number Ops[ParentIdx] = getScope(Var->getScope())->getId(); Ops[FlagsIdx] = transDebugFlags(Var); if (SPIRVWord ArgNumber = Var->getArg()) Ops.push_back(ArgNumber); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {LineIdx, ColumnIdx, FlagsIdx}); return BM->addDebugInfo(SPIRVDebug::LocalVariable, getVoidTy(), Ops); } // DWARF Operations and expressions SPIRVEntry *LLVMToSPIRVDbgTran::transDbgExpression(const DIExpression *Expr) { SPIRVWordVec Operations; for (unsigned I = 0, N = Expr->getNumElements(); I < N; ++I) { using namespace SPIRVDebug::Operand::Operation; auto DWARFOpCode = static_cast(Expr->getElement(I)); SPIRVDebug::ExpressionOpCode OC = SPIRV::DbgExpressionOpCodeMap::map(DWARFOpCode); if (OpCountMap.find(OC) == OpCountMap.end()) report_fatal_error(llvm::Twine("unknown opcode found in DIExpression")); if (OC > SPIRVDebug::Fragment && !(BM->allowExtraDIExpressions() || BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200)) report_fatal_error( llvm::Twine("unsupported opcode found in DIExpression")); unsigned OpCount = OpCountMap[OC]; SPIRVWordVec Op(OpCount); Op[OpCodeIdx] = OC; if (isNonSemanticDebugInfo()) transformToConstant(Op, {OpCodeIdx}); for (unsigned J = 1; J < OpCount; ++J) { Op[J] = Expr->getElement(++I); if (isNonSemanticDebugInfo()) transformToConstant(Op, {J}); } auto *Operation = BM->addDebugInfo(SPIRVDebug::Operation, getVoidTy(), Op); Operations.push_back(Operation->getId()); } return BM->addDebugInfo(SPIRVDebug::Expression, getVoidTy(), Operations); } // Imported entries (C++ using directive) SPIRVEntry * LLVMToSPIRVDbgTran::transDbgImportedEntry(const DIImportedEntity *IE) { using namespace SPIRVDebug::Operand::ImportedEntity; auto Tag = static_cast(IE->getTag()); // FIXME: 'OpenCL/bugged' version is kept because it's hard to remove it // It's W/A for missing 2nd index in OpenCL's implementation const SPIRVWord OffsetIdx = static_cast(isNonSemanticDebugInfo()); SPIRVWordVec Ops(OpenCL::OperandCount - OffsetIdx); Ops[OpenCL::NameIdx] = BM->getString(IE->getName().str())->getId(); Ops[OpenCL::TagIdx] = SPIRV::DbgImportedEntityMap::map(Tag); Ops[OpenCL::SourceIdx - OffsetIdx] = getSource(IE->getFile())->getId(); Ops[OpenCL::EntityIdx - OffsetIdx] = transDbgEntry(IE->getEntity())->getId(); Ops[OpenCL::LineIdx - OffsetIdx] = IE->getLine(); // This version of DIImportedEntity has no column number Ops[OpenCL::ColumnIdx - OffsetIdx] = 0; Ops[OpenCL::ParentIdx - OffsetIdx] = getScope(IE->getScope())->getId(); if (isNonSemanticDebugInfo()) transformToConstant(Ops, {OpenCL::TagIdx, OpenCL::LineIdx - OffsetIdx, OpenCL::ColumnIdx - OffsetIdx}); return BM->addDebugInfo(SPIRVDebug::ImportedEntity, getVoidTy(), Ops); } SPIRVEntry *LLVMToSPIRVDbgTran::transDbgModule(const DIModule *Module) { using namespace SPIRVDebug::Operand::ModuleINTEL; SPIRVWordVec Ops(OperandCount); Ops[NameIdx] = BM->getString(Module->getName().str())->getId(); Ops[SourceIdx] = getSource(Module->getFile())->getId(); Ops[LineIdx] = Module->getLineNo(); Ops[ParentIdx] = getScope(Module->getScope())->getId(); Ops[ConfigMacrosIdx] = BM->getString(Module->getConfigurationMacros().str())->getId(); Ops[IncludePathIdx] = BM->getString(Module->getIncludePath().str())->getId(); Ops[ApiNotesIdx] = BM->getString(Module->getAPINotesFile().str())->getId(); Ops[IsDeclIdx] = Module->getIsDecl(); if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { // The difference in translation of NonSemantic Debug Info and // SPV_INTEL_debug_module extension is that extension allows Line and IsDecl // operands to be Literals, when the non-OpenCL Debug Info allows only IDs // to the constant values. transformToConstant(Ops, {LineIdx, IsDeclIdx}); return BM->addDebugInfo(SPIRVDebug::Module, getVoidTy(), Ops); } BM->addExtension(ExtensionID::SPV_INTEL_debug_module); BM->addCapability(spv::CapabilityDebugInfoModuleINTEL); return BM->addDebugInfo(SPIRVDebug::ModuleINTEL, getVoidTy(), Ops); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/LLVMToSPIRVDbgTran.h000066400000000000000000000201011506646577400224720ustar00rootroot00000000000000//===- LLVMToSPIRVDbgTran.h - Converts LLVM DebugInfo to SPIR-V -*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2018 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements translation of debug info from LLVM metadata to SPIR-V // //===----------------------------------------------------------------------===// #ifndef LLVMTOSPIRVDBGTRAN_HPP_ #define LLVMTOSPIRVDBGTRAN_HPP_ #include "SPIRVModule.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Module.h" using namespace llvm; namespace SPIRV { class LLVMToSPIRVBase; class LLVMToSPIRVDbgTran { public: typedef std::vector SPIRVWordVec; LLVMToSPIRVDbgTran(Module *TM = nullptr, SPIRVModule *TBM = nullptr, LLVMToSPIRVBase *Writer = nullptr) : BM(TBM), M(TM), SPIRVWriter(Writer), VoidT(nullptr), DebugInfoNone(nullptr) {} void transDebugMetadata(); void setModule(Module *Mod) { M = Mod; } // Mixing translation of regular instructions and debug info creates a mess. // To avoid it we translate debug info intrinsics in two steps: // 1. First time we meet debug info intrinsic during translation of a basic // block. At this time we create corresponding SPIRV debug info instruction, // but with dummy operands. Doing so we a) map llvm value to spirv value, // b) get a place for SPIRV debug info intrinsic in SPIRV basic block. // We also remember all debug intrinsics. SPIRVValue *createDebugDeclarePlaceholder(const DbgVariableIntrinsic *DbgDecl, SPIRVBasicBlock *BB); SPIRVValue *createDebugValuePlaceholder(const DbgVariableIntrinsic *DbgValue, SPIRVBasicBlock *BB); private: // 2. After translation of all regular instructions we deal with debug info. // We iterate over debug intrinsics stored on the first step, get its mapped // SPIRV instruction and tweak the operands. void finalizeDebugDeclare(const DbgVariableIntrinsic *DbgDecl); void finalizeDebugValue(const DbgVariableIntrinsic *DbgValue); // Emit DebugScope and OpLine instructions void transLocationInfo(); // Dispatcher SPIRVEntry *transDbgEntry(const MDNode *DIEntry); SPIRVEntry *transDbgEntryImpl(const MDNode *MDN); // Helper methods SPIRVType *getVoidTy(); SPIRVType *getInt32Ty(); SPIRVEntry *getScope(DIScope *SR); SPIRVEntry *getGlobalVariable(const DIGlobalVariable *GV); inline bool isNonSemanticDebugInfo(); void transformToConstant(std::vector &Ops, std::vector Idxs); // No debug info SPIRVEntry *getDebugInfoNone(); SPIRVId getDebugInfoNoneId(); // Compilation unit SPIRVEntry *transDbgCompileUnit(const DICompileUnit *CU); /// The following methods (till the end of the file) implement translation /// of debug instrtuctions described in the spec. // Types SPIRVEntry *transDbgBaseType(const DIBasicType *BT); SPIRVEntry *transDbgPointerType(const DIDerivedType *PT); SPIRVEntry *transDbgQualifiedType(const DIDerivedType *QT); SPIRVEntry *transDbgArrayType(const DICompositeType *AT); SPIRVEntry *transDbgArrayTypeOpenCL(const DICompositeType *AT); SPIRVEntry *transDbgArrayTypeNonSemantic(const DICompositeType *AT); SPIRVEntry *transDbgArrayTypeDynamic(const DICompositeType *AT); SPIRVEntry *transDbgSubrangeType(const DISubrange *ST); SPIRVEntry *transDbgStringType(const DIStringType *ST); SPIRVEntry *transDbgTypeDef(const DIDerivedType *D); SPIRVEntry *transDbgSubroutineType(const DISubroutineType *FT); SPIRVEntry *transDbgEnumType(const DICompositeType *ET); SPIRVEntry *transDbgCompositeType(const DICompositeType *CT); SPIRVEntry *transDbgMemberType(const DIDerivedType *MT); SPIRVEntry *transDbgMemberTypeOpenCL(const DIDerivedType *MT); SPIRVEntry *transDbgMemberTypeNonSemantic(const DIDerivedType *MT); SPIRVEntry *transDbgInheritance(const DIDerivedType *DT); SPIRVEntry *transDbgPtrToMember(const DIDerivedType *DT); // Templates SPIRVEntry *transDbgTemplateParams(DITemplateParameterArray TPA, const SPIRVEntry *Target); SPIRVEntry *transDbgTemplateParameter(const DITemplateParameter *TP); SPIRVEntry * transDbgTemplateTemplateParameter(const DITemplateValueParameter *TP); SPIRVEntry *transDbgTemplateParameterPack(const DITemplateValueParameter *TP); // Global objects SPIRVEntry *transDbgGlobalVariable(const DIGlobalVariable *GV); SPIRVEntry *transDbgFunction(const DISubprogram *Func); SPIRVEntry *transDbgFuncDefinition(SPIRVValue *SPVFunc, SPIRVEntry *DbgFunc); SPIRVEntry *transDbgEntryPoint(const DISubprogram *Func, SPIRVEntry *DbgFunc); // Location information SPIRVEntry *transDbgScope(const DIScope *S); SPIRVEntry *transDebugLoc(const DebugLoc &Loc, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore = nullptr); SPIRVEntry *transDbgInlinedAt(const DILocation *D); SPIRVEntry *transDbgInlinedAtNonSemanticShader200(const DILocation *D); template SPIRVExtInst *getSource(const T *DIEntry); SPIRVEntry *transDbgFileType(const DIFile *F); // Generate instructions recording identifier and file where debug information // was split to void generateBuildIdentifierAndStoragePath(const DICompileUnit *DIEntry); // Local Variables SPIRVEntry *transDbgLocalVariable(const DILocalVariable *Var); // DWARF expressions SPIRVEntry *transDbgExpression(const DIExpression *Expr); // Imported declarations and modules SPIRVEntry *transDbgImportedEntry(const DIImportedEntity *IE); // A module in programming language. Example - Fortran module, clang module. SPIRVEntry *transDbgModule(const DIModule *IE); // Flags SPIRVWord mapDebugFlags(DINode::DIFlags DFlags); SPIRVWord transDebugFlags(const DINode *DN); SPIRVModule *BM; Module *M; LLVMToSPIRVBase *SPIRVWriter; std::unordered_map MDMap; std::unordered_map FileMap; DebugInfoFinder DIF; SPIRVType *VoidT = nullptr; SPIRVType *Int32T = nullptr; SPIRVEntry *DebugInfoNone; std::unordered_map SPIRVCUMap; std::vector DbgDeclareIntrinsics; std::vector DbgValueIntrinsics; SPIRVExtInst *BuildIdentifierInsn{nullptr}; SPIRVExtInst *StoragePathInsn{nullptr}; }; // class LLVMToSPIRVDbgTran } // namespace SPIRV #endif // LLVMTOSPIRVDBGTRAN_HPP_ SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/000077500000000000000000000000001506646577400205515ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/FunctionDescriptor.cpp000066400000000000000000000044651506646577400251120ustar00rootroot00000000000000//===---------------------- FunctionDescriptor.cpp -----------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #include "FunctionDescriptor.h" #include "ParameterType.h" #include namespace SPIR { std::string FunctionDescriptor::nullString() { return std::string(""); } std::string FunctionDescriptor::toString() const { std::stringstream Stream; if (isNull()) { return FunctionDescriptor::nullString(); } Stream << Name << "("; size_t ParamCount = Parameters.size(); if (ParamCount > 0) { for (size_t I = 0; I < ParamCount - 1; ++I) Stream << Parameters[I]->toString() << ", "; Stream << Parameters[ParamCount - 1]->toString(); } Stream << ")"; return Stream.str(); } static bool equal(const TypeVector &L, const TypeVector &R) { if (&L == &R) return true; if (L.size() != R.size()) return false; TypeVector::const_iterator Itl = L.begin(), Itr = R.begin(), Endl = L.end(); while (Itl != Endl) { if (!(*Itl)->equals(*Itr)) return false; ++Itl; ++Itr; } return true; } // // FunctionDescriptor // bool FunctionDescriptor::operator==(const FunctionDescriptor &That) const { if (this == &That) return true; if (Name != That.Name) return false; return equal(Parameters, That.Parameters); } bool FunctionDescriptor::operator<(const FunctionDescriptor &That) const { int StrCmp = Name.compare(That.Name); if (StrCmp) return (StrCmp < 0); size_t Len = Parameters.size(), ThatLen = That.Parameters.size(); if (Len != ThatLen) return Len < ThatLen; TypeVector::const_iterator It = Parameters.begin(), E = Parameters.end(), Thatit = That.Parameters.begin(); while (It != E) { int Cmp = (*It)->toString().compare((*Thatit)->toString()); if (Cmp) return (Cmp < 0); ++Thatit; ++It; } return false; } bool FunctionDescriptor::isNull() const { return (Name.empty() && Parameters.empty()); } FunctionDescriptor FunctionDescriptor::null() { FunctionDescriptor Fd; Fd.Name = ""; return Fd; } } // namespace SPIR SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/FunctionDescriptor.h000066400000000000000000000027741506646577400245600ustar00rootroot00000000000000//===----------------------- FunctionDescriptor.h ------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #ifndef SPIRV_MANGLER_FUNCTIONDESCRIPTOR_H #define SPIRV_MANGLER_FUNCTIONDESCRIPTOR_H #include "ParameterType.h" #include "Refcount.h" #include #include namespace SPIR { typedef std::vector> TypeVector; struct FunctionDescriptor { /// @brief Returns a human readable string representation of the function's /// prototype. /// @returns std::string representing the function's prototype. std::string toString() const; /// The name of the function (stripped). std::string Name; /// Parameter list of the function. TypeVector Parameters; bool operator==(const FunctionDescriptor &) const; /// @brief Enables function descriptors to serve as keys in stl maps. bool operator<(const FunctionDescriptor &) const; bool isNull() const; /// @brief Create a singular value, that represents a 'null' /// FunctionDescriptor. static FunctionDescriptor null(); static std::string nullString(); }; template std::ostream &operator<<(T &O, const SPIR::FunctionDescriptor &Fd) { O << Fd.toString(); return O; } } // namespace SPIR #endif // SPIRV_MANGLER_FUNCTIONDESCRIPTOR_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/Mangler.cpp000066400000000000000000000162261506646577400226510ustar00rootroot00000000000000//===--------------------------- Mangler.cpp -----------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #include "FunctionDescriptor.h" #include "ManglingUtils.h" #include "NameMangleAPI.h" #include "ParameterType.h" #include #include #include #include // According to IA64 name mangling spec, // builtin vector types should not be substituted // This is a workaround till this gets fixed in CLang #define ENABLE_MANGLER_VECTOR_SUBSTITUTION 1 namespace SPIR { class MangleVisitor : public TypeVisitor { public: MangleVisitor(SPIRversion Ver, std::stringstream &S) : TypeVisitor(Ver), Stream(S), SeqId(0) {} // // mangle substitution methods // void mangleSequenceID(unsigned SeqID) { if (SeqID == 1) Stream << '0'; else if (SeqID > 1) { std::string Bstr; std::string Charset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; SeqID--; Bstr.reserve(7); for (; SeqID != 0; SeqID /= 36) Bstr += Charset.substr(SeqID % 36, 1); std::reverse(Bstr.begin(), Bstr.end()); Stream << Bstr; } Stream << '_'; } bool mangleSubstitution(const ParamType *Type, std::string TypeStr) { size_t Fpos; std::stringstream ThistypeStr; ThistypeStr << TypeStr; if ((Fpos = Stream.str().find(TypeStr)) != std::string::npos) { const char *NType; if (const PointerType *P = SPIR::dynCast(Type)) { ThistypeStr << getPointeeMangling(P->getPointee()); } #if defined(ENABLE_MANGLER_VECTOR_SUBSTITUTION) else if (const VectorType *PVec = SPIR::dynCast(Type)) { if ((NType = mangledPrimitiveStringfromName( PVec->getScalarType()->toString()))) ThistypeStr << NType; } #endif std::map::iterator I = Substitutions.find(ThistypeStr.str()); if (I == Substitutions.end()) return false; unsigned SeqID = I->second; Stream << 'S'; mangleSequenceID(SeqID); return true; } return false; } // // Visit methods // MangleError visit(const PrimitiveType *T) override { MangleError Me = MANGLE_SUCCESS; std::string MangledPrimitive = std::string(mangledPrimitiveString(T->getPrimitive())); #if defined(SPIRV_SPIR20_MANGLING_REQUIREMENTS) Stream << MangledPrimitive; #else // Builtin primitives such as int are not substitution candidates, but // all other primitives are. Even though most of these do not appear // repeatedly in builtin function signatures, we need to track them in // the substitution map. if (T->getPrimitive() >= PRIMITIVE_STRUCT_FIRST) { if (!mangleSubstitution(T, MangledPrimitive)) { size_t Index = Stream.str().size(); Stream << MangledPrimitive; recordSubstitution(Stream.str().substr(Index)); } } else { Stream << MangledPrimitive; } #endif return Me; } MangleError visit(const PointerType *P) override { size_t Fpos = Stream.str().size(); MangleError Me = MANGLE_SUCCESS; std::string AttrMangling = getPointerAttributesMangling(P); if (!mangleSubstitution(P, "P" + AttrMangling)) { // A pointee type is substituted when it is a user type, a vector type // (but see a comment in the beginning of this file), a pointer type, // or a primitive type with qualifiers (addr. space and/or CV qualifiers). // So, stream "P", type qualifiers Stream << "P" << AttrMangling; // and the pointee type itself. Me = P->getPointee()->accept(this); // The type qualifiers plus a pointee type is a substitutable entity, but // only when there are qualifiers in the first place. if (!AttrMangling.empty()) recordSubstitution(Stream.str().substr(Fpos + 1)); // The complete pointer type is substitutable as well recordSubstitution(Stream.str().substr(Fpos)); } return Me; } MangleError visit(const VectorType *V) override { size_t Index = Stream.str().size(); std::stringstream TypeStr; TypeStr << "Dv" << V->getLength() << "_"; MangleError Me = MANGLE_SUCCESS; #if defined(ENABLE_MANGLER_VECTOR_SUBSTITUTION) if (!mangleSubstitution(V, TypeStr.str())) #endif { Stream << TypeStr.str(); Me = V->getScalarType()->accept(this); recordSubstitution(Stream.str().substr(Index)); } return Me; } MangleError visit(const AtomicType *P) override { MangleError Me = MANGLE_SUCCESS; size_t Index = Stream.str().size(); const char *TypeStr = "U7_Atomic"; if (!mangleSubstitution(P, TypeStr)) { Stream << TypeStr; Me = P->getBaseType()->accept(this); recordSubstitution(Stream.str().substr(Index)); } return Me; } MangleError visit(const BlockType *P) override { Stream << "U" << "13block_pointerFv"; if (P->getNumOfParams() == 0) Stream << "v"; else for (unsigned int I = 0; I < P->getNumOfParams(); ++I) { MangleError Err = P->getParam(I)->accept(this); if (Err != MANGLE_SUCCESS) { return Err; } } Stream << "E"; // "Add" the function type (FvvE) and U13block_pointerFvvE to the // substitution table. We don't actually substitute this if it's present, // but since the block type only occurs at most once in any function we care // about, this should be sufficient. SeqId += 2; return MANGLE_SUCCESS; } MangleError visit(const UserDefinedType *PTy) override { size_t Index = Stream.str().size(); std::string Name = PTy->toString(); if (!mangleSubstitution(PTy, Name)) { Stream << Name.size() << Name; recordSubstitution(Stream.str().substr(Index)); } return MANGLE_SUCCESS; } private: void recordSubstitution(const std::string &Str) { Substitutions[Str] = SeqId++; } // Holds the mangled string representing the prototype of the function. std::stringstream &Stream; unsigned SeqId; std::map Substitutions; }; // // NameMangler // NameMangler::NameMangler(SPIRversion Version) : SpirVersion(Version) {} MangleError NameMangler::mangle(const FunctionDescriptor &Fd, std::string &MangledName) { if (Fd.isNull()) { MangledName.assign(FunctionDescriptor::nullString()); return MANGLE_NULL_FUNC_DESCRIPTOR; } std::stringstream Ret; Ret << "_Z" << Fd.Name.length() << Fd.Name; MangleVisitor Visitor(SpirVersion, Ret); for (unsigned int I = 0; I < Fd.Parameters.size(); ++I) { MangleError Err = Fd.Parameters[I]->accept(&Visitor); if (Err == MANGLE_TYPE_NOT_SUPPORTED) { MangledName.assign("Type "); MangledName.append(Fd.Parameters[I]->toString()); MangledName.append(" is not supported in "); std::string Ver = getSPIRVersionAsString(SpirVersion); MangledName.append(Ver); return Err; } } MangledName.assign(Ret.str()); return MANGLE_SUCCESS; } } // namespace SPIR SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/ManglingUtils.cpp000066400000000000000000000305611506646577400240370ustar00rootroot00000000000000//===------------------------- ManglingUtils.cpp -------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #include "ManglingUtils.h" namespace SPIR { // String represenration for the primitive types. static const char *PrimitiveNames[PRIMITIVE_NUM] = { "bool", "uchar", "char", "ushort", "short", "uint", "int", "ulong", "long", "half", "float", "double", "__bf16", "void", "...", "image1d_ro_t", "image1d_array_ro_t", "image1d_buffer_ro_t", "image2d_ro_t", "image2d_array_ro_t", "image2d_depth_ro_t", "image2d_array_depth_ro_t", "image2d_msaa_ro_t", "image2d_array_msaa_ro_t", "image2d_msaa_depth_ro_t", "image2d_array_msaa_depth_ro_t", "image3d_ro_t", "image1d_wo_t", "image1d_array_wo_t", "image1d_buffer_wo_t", "image2d_wo_t", "image2d_array_wo_t", "image2d_depth_wo_t", "image2d_array_depth_wo_t", "image2d_msaa_wo_t", "image2d_array_msaa_wo_t", "image2d_msaa_depth_wo_t", "image2d_array_msaa_depth_wo_t", "image3d_wo_t", "image1d_rw_t", "image1d_array_rw_t", "image1d_buffer_rw_t", "image2d_rw_t", "image2d_array_rw_t", "image2d_depth_rw_t", "image2d_array_depth_rw_t", "image2d_msaa_rw_t", "image2d_array_msaa_rw_t", "image2d_msaa_depth_rw_t", "image2d_array_msaa_depth_rw_t", "image3d_rw_t", "event_t", "pipe_ro_t", "pipe_wo_t", "reserve_id_t", "queue_t", "ndrange_t", "clk_event_t", "sampler_t", "kernel_enqueue_flags_t", "clk_profiling_info", "memory_order", "memory_scope", "intel_sub_group_avc_mce_payload_t", "intel_sub_group_avc_ime_payload_t", "intel_sub_group_avc_ref_payload_t", "intel_sub_group_avc_sic_payload_t", "intel_sub_group_avc_mce_result_t", "intel_sub_group_avc_ime_result_t", "intel_sub_group_avc_ref_result_t", "intel_sub_group_avc_sic_result_t", "intel_sub_group_avc_ime_result_single_reference_streamout_t", "intel_sub_group_avc_ime_result_dual_reference_streamout_t", "intel_sub_group_avc_ime_result_single_reference_streamin_t", "intel_sub_group_avc_ime_result_dual_reference_streamin_t" }; // clang-format off const char *MangledTypes[PRIMITIVE_NUM] = { "b", // BOOL "h", // UCHAR "c", // CHAR "t", // USHORT "s", // SHORT "j", // UINT "i", // INT "m", // ULONG "l", // LONG "Dh", // HALF "f", // FLOAT "d", // DOUBLE "DF16b", // __BF16 "v", // VOID "z", // VarArg "14ocl_image1d_ro", // PRIMITIVE_IMAGE1D_RO_T "20ocl_image1d_array_ro", // PRIMITIVE_IMAGE1D_ARRAY_RO_T "21ocl_image1d_buffer_ro", // PRIMITIVE_IMAGE1D_BUFFER_RO_T "14ocl_image2d_ro", // PRIMITIVE_IMAGE2D_RO_T "20ocl_image2d_array_ro", // PRIMITIVE_IMAGE2D_ARRAY_RO_T "20ocl_image2d_depth_ro", // PRIMITIVE_IMAGE2D_DEPTH_RO_T "26ocl_image2d_array_depth_ro", // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T "19ocl_image2d_msaa_ro", // PRIMITIVE_IMAGE2D_MSAA_RO_T "25ocl_image2d_array_msaa_ro", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T "25ocl_image2d_msaa_depth_ro", // PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T "31ocl_image2d_array_msaa_depth_ro", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T "14ocl_image3d_ro", // PRIMITIVE_IMAGE3D_RO_T "14ocl_image1d_wo", // PRIMITIVE_IMAGE1D_WO_T "20ocl_image1d_array_wo", // PRIMITIVE_IMAGE1D_ARRAY_WO_T "21ocl_image1d_buffer_wo", // PRIMITIVE_IMAGE1D_BUFFER_WO_T "14ocl_image2d_wo", // PRIMITIVE_IMAGE2D_WO_T "20ocl_image2d_array_wo", // PRIMITIVE_IMAGE2D_ARRAY_WO_T "20ocl_image2d_depth_wo", // PRIMITIVE_IMAGE2D_DEPTH_WO_T "26ocl_image2d_array_depth_wo", // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T "19ocl_image2d_msaa_wo", // PRIMITIVE_IMAGE2D_MSAA_WO_T "25ocl_image2d_array_msaa_wo", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T "25ocl_image2d_msaa_depth_wo", // PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T "31ocl_image2d_array_msaa_depth_wo", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T "14ocl_image3d_wo", // PRIMITIVE_IMAGE3D_WO_T "14ocl_image1d_rw", // PRIMITIVE_IMAGE1D_RW_T "20ocl_image1d_array_rw", // PRIMITIVE_IMAGE1D_ARRAY_RW_T "21ocl_image1d_buffer_rw", // PRIMITIVE_IMAGE1D_BUFFER_RW_T "14ocl_image2d_rw", // PRIMITIVE_IMAGE2D_RW_T "20ocl_image2d_array_rw", // PRIMITIVE_IMAGE2D_ARRAY_RW_T "20ocl_image2d_depth_rw", // PRIMITIVE_IMAGE2D_DEPTH_RW_T "26ocl_image2d_array_depth_rw", // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T "19ocl_image2d_msaa_rw", // PRIMITIVE_IMAGE2D_MSAA_RW_T "25ocl_image2d_array_msaa_rw", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T "25ocl_image2d_msaa_depth_rw", // PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T "31ocl_image2d_array_msaa_depth_rw", // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T "14ocl_image3d_rw", // PRIMITIVE_IMAGE3D_RW_T "9ocl_event", // PRIMITIVE_EVENT_T "11ocl_pipe_ro", // PRIMITIVE_PIPE_RO_T "11ocl_pipe_wo", // PRIMITIVE_PIPE_WO_T "13ocl_reserveid", // PRIMITIVE_RESERVE_ID_T "9ocl_queue", // PRIMITIVE_QUEUE_T "9ndrange_t", // PRIMITIVE_NDRANGE_T "12ocl_clkevent", // PRIMITIVE_CLK_EVENT_T "11ocl_sampler", // PRIMITIVE_SAMPLER_T "i", // PRIMITIVE_KERNEL_ENQUEUE_FLAGS_T "i", // PRIMITIVE_CLK_PROFILING_INFO #if defined(SPIRV_SPIR20_MANGLING_REQUIREMENTS) "i", // PRIMITIVE_MEMORY_ORDER "i", // PRIMITIVE_MEMORY_SCOPE #else "12memory_order", // PRIMITIVE_MEMORY_ORDER "12memory_scope", // PRIMITIVE_MEMORY_SCOPE #endif "37ocl_intel_sub_group_avc_mce_payload_t", // PRIMITIVE_SUB_GROUP_AVC_MCE_PAYLOAD_T "37ocl_intel_sub_group_avc_ime_payload_t", // PRIMITIVE_SUB_GROUP_AVC_IME_PAYLOAD_T "37ocl_intel_sub_group_avc_ref_payload_t", // PRIMITIVE_SUB_GROUP_AVC_REF_PAYLOAD_T "37ocl_intel_sub_group_avc_sic_payload_t", // PRIMITIVE_SUB_GROUP_AVC_SIC_PAYLOAD_T "36ocl_intel_sub_group_avc_mce_result_t", // PRIMITIVE_SUB_GROUP_AVC_MCE_RESULT_T "36ocl_intel_sub_group_avc_ime_result_t", // PRIMITIVE_SUB_GROUP_AVC_IME_RESULT_T "36ocl_intel_sub_group_avc_ref_result_t", // PRIMITIVE_SUB_GROUP_AVC_REF_RESULT_T "36ocl_intel_sub_group_avc_sic_result_t", // PRIMITIVE_SUB_GROUP_AVC_REF_RESULT_T "63ocl_intel_sub_group_avc_ime_result_single_reference_streamout_t", // PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMOUT_T "61ocl_intel_sub_group_avc_ime_result_dual_reference_streamout_t", // PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMOUT_T "55ocl_intel_sub_group_avc_ime_single_reference_streamin_t", // PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMIN_T "53ocl_intel_sub_group_avc_ime_dual_reference_streamin_t" // PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMIN_T }; // clang-format on const char *ReadableAttribute[ATTR_NUM] = { "restrict", "volatile", "const", "__private", "__global", "__constant", "__local", "__generic", }; const char *MangledAttribute[ATTR_NUM] = { "r", "V", "K", "", "U3AS1", "U3AS2", "U3AS3", "U3AS4", }; // SPIR supported version - stated version is oldest supported version. static const SPIRversion PrimitiveSupportedVersions[PRIMITIVE_NUM] = { SPIR12, // BOOL SPIR12, // UCHAR SPIR12, // CHAR SPIR12, // USHORT SPIR12, // SHORT SPIR12, // UINT SPIR12, // INT SPIR12, // ULONG SPIR12, // LONG SPIR12, // HALF SPIR12, // FLOAT SPIR12, // DOUBLE SPIR12, // __BF16 SPIR12, // VOID SPIR12, // VarArg SPIR12, // PRIMITIVE_IMAGE1D_RO_T SPIR12, // PRIMITIVE_IMAGE1D_ARRAY_RO_T SPIR12, // PRIMITIVE_IMAGE1D_BUFFER_RO_T SPIR12, // PRIMITIVE_IMAGE2D_RO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_RO_T SPIR12, // PRIMITIVE_IMAGE2D_DEPTH_RO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_RO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T SPIR12, // PRIMITIVE_IMAGE3D_RO_T SPIR12, // PRIMITIVE_IMAGE1D_WO_T SPIR12, // PRIMITIVE_IMAGE1D_ARRAY_WO_T SPIR12, // PRIMITIVE_IMAGE1D_BUFFER_WO_T SPIR12, // PRIMITIVE_IMAGE2D_WO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_WO_T SPIR12, // PRIMITIVE_IMAGE2D_DEPTH_WO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_WO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T SPIR12, // PRIMITIVE_IMAGE3D_WO_T SPIR12, // PRIMITIVE_IMAGE1D_RW_T SPIR12, // PRIMITIVE_IMAGE1D_ARRAY_RW_T SPIR12, // PRIMITIVE_IMAGE1D_BUFFER_RW_T SPIR12, // PRIMITIVE_IMAGE2D_RW_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_RW_T SPIR12, // PRIMITIVE_IMAGE2D_DEPTH_RW_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_RW_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T SPIR12, // PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T SPIR12, // PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T SPIR12, // PRIMITIVE_IMAGE3D_RW_T SPIR12, // PRIMITIVE_EVENT_T SPIR20, // PRIMITIVE_PIPE_RO_T SPIR20, // PRIMITIVE_PIPE_WO_T SPIR20, // PRIMITIVE_RESERVE_ID_T SPIR20, // PRIMITIVE_QUEUE_T SPIR20, // PRIMITIVE_NDRANGE_T SPIR20, // PRIMITIVE_CLK_EVENT_T SPIR12 // PRIMITIVE_SAMPLER_T }; const char *mangledPrimitiveString(TypePrimitiveEnum T) { return MangledTypes[T]; } const char *readablePrimitiveString(TypePrimitiveEnum T) { return PrimitiveNames[T]; } const char *getMangledAttribute(TypeAttributeEnum Attribute) { return MangledAttribute[Attribute]; } const char *getReadableAttribute(TypeAttributeEnum Attribute) { return ReadableAttribute[Attribute]; } SPIRversion getSupportedVersion(TypePrimitiveEnum T) { return PrimitiveSupportedVersions[T]; } const char *mangledPrimitiveStringfromName(std::string Type) { for (size_t I = 0; I < (sizeof(PrimitiveNames) / sizeof(PrimitiveNames[0])); I++) if (Type == PrimitiveNames[I]) return MangledTypes[I]; return NULL; } std::string getPointerAttributesMangling(const PointerType *P) { std::string QualStr; QualStr += getMangledAttribute((P->getAddressSpace())); for (unsigned int I = ATTR_QUALIFIER_FIRST; I <= ATTR_QUALIFIER_LAST; I++) { TypeAttributeEnum Qualifier = (TypeAttributeEnum)I; if (P->hasQualifier(Qualifier)) { QualStr += getMangledAttribute(Qualifier); } } return QualStr; } std::string getPointeeMangling(RefParamType Pointee) { std::string Mangling; while (const PointerType *P = SPIR::dynCast(Pointee)) { Mangling += "P" + getPointerAttributesMangling(P); Pointee = P->getPointee(); } if (const UserDefinedType *U = SPIR::dynCast(Pointee)) { std::string Name = U->toString(); Mangling += std::to_string(Name.size()) + Name; } else if (const char *PrimitiveMangling = mangledPrimitiveStringfromName(Pointee->toString())) { Mangling += PrimitiveMangling; } return Mangling; } const char *getSPIRVersionAsString(SPIRversion Version) { switch (Version) { case SPIR12: return "SPIR 1.2"; case SPIR20: return "SPIR 2.0"; } assert(false && "Unknown SPIR Version"); return "Unknown SPIR Version"; } } // namespace SPIR SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/ManglingUtils.h000066400000000000000000000021231506646577400234750ustar00rootroot00000000000000//===------------------------- ManglingUtils.h ---------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #ifndef SPIRV_MANGLER_MANGLINGUTILS_H #define SPIRV_MANGLER_MANGLINGUTILS_H #include "ParameterType.h" namespace SPIR { const char *mangledPrimitiveString(TypePrimitiveEnum Primitive); const char *readablePrimitiveString(TypePrimitiveEnum Primitive); const char *getMangledAttribute(TypeAttributeEnum Attribute); const char *getReadableAttribute(TypeAttributeEnum Attribute); SPIRversion getSupportedVersion(TypePrimitiveEnum T); const char *getSPIRVersionAsString(SPIRversion Version); const char *mangledPrimitiveStringfromName(std::string Type); std::string getPointerAttributesMangling(const PointerType *P); std::string getPointeeMangling(RefParamType Pointee); } // namespace SPIR #endif // SPIRV_MANGLER_MANGLINGUTILS_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/NameMangleAPI.h000066400000000000000000000025211506646577400232600ustar00rootroot00000000000000//===------------------------- NameMangleAPI.h ---------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #ifndef SPIRV_MANGLER_NAMEMANGLEAPI_H #define SPIRV_MANGLER_NAMEMANGLEAPI_H #include "FunctionDescriptor.h" #include namespace SPIR { struct NameMangler { /// @brief Constructor. /// @param SPIRversion spir version to mangle according to. NameMangler(SPIRversion); /// @brief Converts the given function descriptor to string that represents /// the function's prototype. /// The mangling algorithm is based on Itanium mangling algorithm /// (http://sourcery.mentor.com/public/cxx-abi/abi.html#mangling), with /// SPIR extensions. /// @param FunctionDescriptor function to be mangled. /// @param std::string the mangled name if the mangling succeeds, /// the error otherwise. /// @return MangleError enum representing the status - success or the error. MangleError mangle(const FunctionDescriptor &, std::string &); private: SPIRversion SpirVersion; }; } // namespace SPIR #endif // SPIRV_MANGLER_NAMEMANGLEAPI_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/ParameterType.cpp000066400000000000000000000141731506646577400240450ustar00rootroot00000000000000//===------------------------ ParameterType.cpp --------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #include "ParameterType.h" #include "ManglingUtils.h" #include #include #include namespace SPIR { // // Primitive Type // PrimitiveType::PrimitiveType(TypePrimitiveEnum Primitive) : ParamType(TYPE_ID_PRIMITIVE), Primitive(Primitive) {} MangleError PrimitiveType::accept(TypeVisitor *Visitor) const { if (getSupportedVersion(this->getPrimitive()) >= SPIR20 && Visitor->SpirVer < SPIR20) { return MANGLE_TYPE_NOT_SUPPORTED; } return Visitor->visit(this); } std::string PrimitiveType::toString() const { assert((Primitive >= PRIMITIVE_FIRST && Primitive <= PRIMITIVE_LAST) && "illegal primitive"); std::stringstream MyName; MyName << readablePrimitiveString(Primitive); return MyName.str(); } bool PrimitiveType::equals(const ParamType *Type) const { const PrimitiveType *P = SPIR::dynCast(Type); return P && (Primitive == P->Primitive); } // // Pointer Type // PointerType::PointerType(const RefParamType Type) : ParamType(TYPE_ID_POINTER), PType(Type) { for (unsigned int I = ATTR_QUALIFIER_FIRST; I <= ATTR_QUALIFIER_LAST; I++) { setQualifier((TypeAttributeEnum)I, false); } AddressSpace = ATTR_PRIVATE; } MangleError PointerType::accept(TypeVisitor *Visitor) const { return Visitor->visit(this); } void PointerType::setAddressSpace(TypeAttributeEnum Attr) { if (Attr < ATTR_ADDR_SPACE_FIRST || Attr > ATTR_ADDR_SPACE_LAST) { return; } AddressSpace = Attr; } TypeAttributeEnum PointerType::getAddressSpace() const { return AddressSpace; } void PointerType::setQualifier(TypeAttributeEnum Qual, bool Enabled) { if (Qual < ATTR_QUALIFIER_FIRST || Qual > ATTR_QUALIFIER_LAST) { return; } Qualifiers[Qual - ATTR_QUALIFIER_FIRST] = Enabled; } bool PointerType::hasQualifier(TypeAttributeEnum Qual) const { if (Qual < ATTR_QUALIFIER_FIRST || Qual > ATTR_QUALIFIER_LAST) { return false; } return Qualifiers[Qual - ATTR_QUALIFIER_FIRST]; } std::string PointerType::toString() const { std::stringstream MyName; for (unsigned int I = ATTR_QUALIFIER_FIRST; I <= ATTR_QUALIFIER_LAST; I++) { TypeAttributeEnum Qual = (TypeAttributeEnum)I; if (hasQualifier(Qual)) { MyName << getReadableAttribute(Qual) << " "; } } MyName << getReadableAttribute(TypeAttributeEnum(AddressSpace)) << " "; MyName << getPointee()->toString() << " *"; return MyName.str(); } bool PointerType::equals(const ParamType *Type) const { const PointerType *P = SPIR::dynCast(Type); if (!P) { return false; } if (getAddressSpace() != P->getAddressSpace()) { return false; } for (unsigned int I = ATTR_QUALIFIER_FIRST; I <= ATTR_QUALIFIER_LAST; I++) { TypeAttributeEnum Qual = (TypeAttributeEnum)I; if (hasQualifier(Qual) != P->hasQualifier(Qual)) { return false; } } return (*getPointee()).equals(&*(P->getPointee())); } // // Vector Type // VectorType::VectorType(const RefParamType Type, int Len) : ParamType(TYPE_ID_VECTOR), PType(Type), Len(Len) {} MangleError VectorType::accept(TypeVisitor *Visitor) const { return Visitor->visit(this); } std::string VectorType::toString() const { std::stringstream MyName; MyName << getScalarType()->toString(); MyName << Len; return MyName.str(); } bool VectorType::equals(const ParamType *Type) const { const VectorType *PVec = SPIR::dynCast(Type); return PVec && (Len == PVec->Len) && (*getScalarType()).equals(&*(PVec->getScalarType())); } // // Atomic Type // AtomicType::AtomicType(const RefParamType Type) : ParamType(TYPE_ID_ATOMIC), PType(Type) {} MangleError AtomicType::accept(TypeVisitor *Visitor) const { if (Visitor->SpirVer < SPIR20) { return MANGLE_TYPE_NOT_SUPPORTED; } return Visitor->visit(this); } std::string AtomicType::toString() const { std::stringstream MyName; MyName << "atomic_" << getBaseType()->toString(); return MyName.str(); } bool AtomicType::equals(const ParamType *Type) const { const AtomicType *A = dynCast(Type); return (A && (*getBaseType()).equals(&*(A->getBaseType()))); } // // Block Type // BlockType::BlockType() : ParamType(TYPE_ID_BLOCK) {} MangleError BlockType::accept(TypeVisitor *Visitor) const { if (Visitor->SpirVer < SPIR20) { return MANGLE_TYPE_NOT_SUPPORTED; } return Visitor->visit(this); } std::string BlockType::toString() const { std::stringstream MyName; MyName << "void ("; for (unsigned int I = 0; I < getNumOfParams(); ++I) { if (I > 0) MyName << ", "; MyName << Params[I]->toString(); } MyName << ")*"; return MyName.str(); } bool BlockType::equals(const ParamType *Type) const { const BlockType *PBlock = dynCast(Type); if (!PBlock || getNumOfParams() != PBlock->getNumOfParams()) { return false; } for (unsigned int I = 0; I < getNumOfParams(); ++I) { if (!getParam(I)->equals(&*PBlock->getParam(I))) { return false; } } return true; } // // User Defined Type // UserDefinedType::UserDefinedType(const std::string &Name) : ParamType(TYPE_ID_STRUCTURE), Name(Name) {} MangleError UserDefinedType::accept(TypeVisitor *Visitor) const { return Visitor->visit(this); } std::string UserDefinedType::toString() const { std::stringstream MyName; MyName << Name; return MyName.str(); } bool UserDefinedType::equals(const ParamType *PType) const { const UserDefinedType *PTy = SPIR::dynCast(PType); return PTy && (Name == PTy->Name); } // // Static enums // const TypeEnum PrimitiveType::EnumTy = TYPE_ID_PRIMITIVE; const TypeEnum PointerType::EnumTy = TYPE_ID_POINTER; const TypeEnum VectorType::EnumTy = TYPE_ID_VECTOR; const TypeEnum AtomicType::EnumTy = TYPE_ID_ATOMIC; const TypeEnum BlockType::EnumTy = TYPE_ID_BLOCK; const TypeEnum UserDefinedType::EnumTy = TYPE_ID_STRUCTURE; } // namespace SPIR SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/ParameterType.h000066400000000000000000000373051506646577400235140ustar00rootroot00000000000000//===------------------------- ParameterType.h ---------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation. */ #ifndef SPIRV_MANGLER_PARAMETERTYPE_H #define SPIRV_MANGLER_PARAMETERTYPE_H #include "Refcount.h" #include #include // The Type class hierarchy models the different types in OCL. namespace SPIR { // Supported SPIR versions enum SPIRversion { SPIR12 = 1, SPIR20 = 2 }; // Error Status values enum MangleError { MANGLE_SUCCESS, MANGLE_TYPE_NOT_SUPPORTED, MANGLE_NULL_FUNC_DESCRIPTOR }; enum TypePrimitiveEnum { PRIMITIVE_FIRST, PRIMITIVE_BOOL = PRIMITIVE_FIRST, PRIMITIVE_UCHAR, PRIMITIVE_CHAR, PRIMITIVE_USHORT, PRIMITIVE_SHORT, PRIMITIVE_UINT, PRIMITIVE_INT, PRIMITIVE_ULONG, PRIMITIVE_LONG, PRIMITIVE_HALF, PRIMITIVE_FLOAT, PRIMITIVE_DOUBLE, PRIMITIVE_BFLOAT, PRIMITIVE_VOID, PRIMITIVE_VAR_ARG, PRIMITIVE_STRUCT_FIRST, PRIMITIVE_IMAGE1D_RO_T = PRIMITIVE_STRUCT_FIRST, PRIMITIVE_IMAGE1D_ARRAY_RO_T, PRIMITIVE_IMAGE1D_BUFFER_RO_T, PRIMITIVE_IMAGE2D_RO_T, PRIMITIVE_IMAGE2D_ARRAY_RO_T, PRIMITIVE_IMAGE2D_DEPTH_RO_T, PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T, PRIMITIVE_IMAGE2D_MSAA_RO_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T, PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T, PRIMITIVE_IMAGE3D_RO_T, PRIMITIVE_IMAGE1D_WO_T, PRIMITIVE_IMAGE1D_ARRAY_WO_T, PRIMITIVE_IMAGE1D_BUFFER_WO_T, PRIMITIVE_IMAGE2D_WO_T, PRIMITIVE_IMAGE2D_ARRAY_WO_T, PRIMITIVE_IMAGE2D_DEPTH_WO_T, PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T, PRIMITIVE_IMAGE2D_MSAA_WO_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T, PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T, PRIMITIVE_IMAGE3D_WO_T, PRIMITIVE_IMAGE1D_RW_T, PRIMITIVE_IMAGE1D_ARRAY_RW_T, PRIMITIVE_IMAGE1D_BUFFER_RW_T, PRIMITIVE_IMAGE2D_RW_T, PRIMITIVE_IMAGE2D_ARRAY_RW_T, PRIMITIVE_IMAGE2D_DEPTH_RW_T, PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T, PRIMITIVE_IMAGE2D_MSAA_RW_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T, PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T, PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T, PRIMITIVE_IMAGE3D_RW_T, PRIMITIVE_EVENT_T, PRIMITIVE_PIPE_RO_T, PRIMITIVE_PIPE_WO_T, PRIMITIVE_RESERVE_ID_T, PRIMITIVE_QUEUE_T, PRIMITIVE_NDRANGE_T, PRIMITIVE_CLK_EVENT_T, PRIMITIVE_STRUCT_LAST = PRIMITIVE_CLK_EVENT_T, PRIMITIVE_SAMPLER_T, PRIMITIVE_KERNEL_ENQUEUE_FLAGS_T, PRIMITIVE_CLK_PROFILING_INFO, PRIMITIVE_MEMORY_ORDER, PRIMITIVE_MEMORY_SCOPE, PRIMITIVE_SUB_GROUP_AVC_MCE_PAYLOAD_T, PRIMITIVE_SUB_GROUP_AVC_IME_PAYLOAD_T, PRIMITIVE_SUB_GROUP_AVC_REF_PAYLOAD_T, PRIMITIVE_SUB_GROUP_AVC_SIC_PAYLOAD_T, PRIMITIVE_SUB_GROUP_AVC_MCE_RESULT_T, PRIMITIVE_SUB_GROUP_AVC_IME_RESULT_T, PRIMITIVE_SUB_GROUP_AVC_REF_RESULT_T, PRIMITIVE_SUB_GROUP_AVC_SIC_RESULT_T, PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMOUT_T, PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMOUT_T, PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMIN_T, PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMIN_T, PRIMITIVE_LAST = PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMIN_T, PRIMITIVE_NONE, // Keep this at the end. PRIMITIVE_NUM = PRIMITIVE_NONE }; enum TypeEnum { TYPE_ID_PRIMITIVE, TYPE_ID_POINTER, TYPE_ID_VECTOR, TYPE_ID_ATOMIC, TYPE_ID_BLOCK, TYPE_ID_STRUCTURE }; enum TypeAttributeEnum { ATTR_QUALIFIER_FIRST = 0, ATTR_RESTRICT = ATTR_QUALIFIER_FIRST, ATTR_VOLATILE, ATTR_CONST, ATTR_QUALIFIER_LAST = ATTR_CONST, ATTR_ADDR_SPACE_FIRST, ATTR_PRIVATE = ATTR_ADDR_SPACE_FIRST, ATTR_GLOBAL, ATTR_CONSTANT, ATTR_LOCAL, ATTR_GENERIC, ATTR_GLOBAL_DEVICE, ATTR_GLOBAL_HOST, ATTR_ADDR_SPACE_LAST = ATTR_GLOBAL_HOST, ATTR_NONE, ATTR_NUM = ATTR_NONE }; // Forward declaration for abstract structure. struct ParamType; typedef RefCount RefParamType; // Forward declaration for abstract structure. struct TypeVisitor; struct ParamType { /// @brief Constructor. /// @param TypeEnum type id. ParamType(TypeEnum TypeId) : TypeId(TypeId){}; /// @brief Destructor. virtual ~ParamType(){}; /// Abstract Methods /// /// @brief Visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor. virtual MangleError accept(TypeVisitor *) const = 0; /// @brief Returns a string representation of the underlying type. /// @return type as string. virtual std::string toString() const = 0; /// @brief Returns true if given param type is equal to this type. /// @param ParamType given param type. /// @return true if given param type is equal to this type and false /// otherwise. virtual bool equals(const ParamType *) const = 0; /// Common Base-Class Methods /// /// @brief Returns type id of underlying type. /// @return type id. TypeEnum getTypeId() const { return TypeId; } private: // @brief Default Constructor. ParamType(); protected: /// An enumeration to identify the type id of this instance. TypeEnum TypeId; }; struct PrimitiveType : public ParamType { /// An enumeration to identify the type id of this class. const static TypeEnum EnumTy; /// @brief Constructor. /// @param TypePrimitiveEnum primitive id. PrimitiveType(TypePrimitiveEnum); /// Implementation of Abstract Methods /// /// @brief Visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor. MangleError accept(TypeVisitor *) const override; /// @brief Returns a string representation of the underlying type. /// @return type as string. std::string toString() const override; /// @brief Returns true if given param type is equal to this type. /// @param ParamType given param type. /// @return true if given param type is equal to this type and false /// otherwise. bool equals(const ParamType *) const override; /// Non-Common Methods /// /// @brief Returns the primitive enumeration of the type. /// @return primitive type. TypePrimitiveEnum getPrimitive() const { return Primitive; } protected: /// An enumeration to identify the primitive type. TypePrimitiveEnum Primitive; }; struct PointerType : public ParamType { /// An enumeration to identify the type id of this class. const static TypeEnum EnumTy; /// @brief Constructor. /// @param RefParamType the type of pointee (that the pointer points at). PointerType(const RefParamType Type); /// Implementation of Abstract Methods /// /// @brief Visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor MangleError accept(TypeVisitor *) const override; /// @brief Returns a string representation of the underlying type. /// @return type as string. std::string toString() const override; /// @brief Returns true if given param type is equal to this type. /// @param ParamType given param type. /// @return true if given param type is equal to this type and false /// otherwise. bool equals(const ParamType *) const override; /// Non-Common Methods /// /// @brief Returns the type the pointer is pointing at. /// @return pointee type. const RefParamType &getPointee() const { return PType; } /// @brief Sets the address space attribute - default is __private /// @param TypeAttributeEnum address space attribute id. void setAddressSpace(TypeAttributeEnum Attr); /// @brief Returns the pointer's address space. /// @return pointer's address space. TypeAttributeEnum getAddressSpace() const; /// @brief Adds or removes a pointer's qualifier. /// @param TypeAttributeEnum qual - qualifier to add/remove. /// @param bool enabled - true if qualifier should exist false otherwise. /// default is set to false. void setQualifier(TypeAttributeEnum Qual, bool Enabled); /// @brief Checks if the pointer has a certain qualifier. /// @param TypeAttributeEnum qual - qualifier to check. /// @return true if the qualifier exists and false otherwise. bool hasQualifier(TypeAttributeEnum Qual) const; private: /// The type this pointer is pointing at. RefParamType PType; /// Array of the pointer's enabled type qualifiers. bool Qualifiers[ATTR_QUALIFIER_LAST - ATTR_QUALIFIER_FIRST + 1]; /// Pointer's address space. TypeAttributeEnum AddressSpace; }; struct VectorType : public ParamType { /// An enumeration to identify the type id of this class. const static TypeEnum EnumTy; /// @brief Constructor. /// @param RefParamType the type of each scalar element in the vector. /// @param int the length of the vector. VectorType(const RefParamType Type, int Len); /// Implementation of Abstract Methods /// /// @brief Visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor. MangleError accept(TypeVisitor *) const override; /// @brief Returns a string representation of the underlying type. /// @return type as string. std::string toString() const override; /// @brief Returns true if given param type is equal to this type. /// @param ParamType given param type. /// @return true if given param type is equal to this type and false /// otherwise. bool equals(const ParamType *) const override; /// Non-Common Methods /// /// @brief Returns the type the vector is packing. /// @return scalar type. const RefParamType &getScalarType() const { return PType; } /// @brief Returns the length of the vector type. /// @return vector type length. int getLength() const { return Len; } private: /// The scalar type of this vector type. RefParamType PType; /// The length of the vector. int Len; }; struct AtomicType : public ParamType { /// an enumeration to identify the type id of this class const static TypeEnum EnumTy; /// @brief Constructor /// @param RefParamType the type refernced as atomic. AtomicType(const RefParamType Type); /// Implementation of Abstract Methods /// /// @brief visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor MangleError accept(TypeVisitor *) const override; /// @brief returns a string representation of the underlying type. /// @return type as string std::string toString() const override; /// @brief returns true if given param type is equal to this type. /// @param ParamType given param type /// @return true if given param type is equal to this type and false otherwise bool equals(const ParamType *) const override; /// Non-Common Methods /// /// @brief returns the base type of the atomic parameter. /// @return base type const RefParamType &getBaseType() const { return PType; } private: /// the type this pointer is pointing at RefParamType PType; }; struct BlockType : public ParamType { /// an enumeration to identify the type id of this class const static TypeEnum EnumTy; ///@brief Constructor BlockType(); /// Implementation of Abstract Methods /// /// @brief visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor MangleError accept(TypeVisitor *) const override; /// @brief returns a string representation of the underlying type. /// @return type as string std::string toString() const override; /// @brief returns true if given param type is equal to this type. /// @param ParamType given param type /// @return true if given param type is equal to this type and false otherwise bool equals(const ParamType *) const override; /// Non-Common Methods /// /// @brief returns the number of parameters of the block. /// @return parameters count unsigned int getNumOfParams() const { return (unsigned int)Params.size(); } ///@brief returns the type of parameter "index" of the block. // @param index the sequential number of the queried parameter ///@return parameter type const RefParamType &getParam(unsigned int Index) const { assert(Params.size() > Index && "index is OOB"); return Params[Index]; } ///@brief set the type of parameter "index" of the block. // @param index the sequential number of the queried parameter // @param type the parameter type void setParam(unsigned int Index, RefParamType Type) { if (Index < getNumOfParams()) { Params[Index] = Type; } else if (Index == getNumOfParams()) { Params.push_back(Type); } else { assert(false && "index is OOB"); } } protected: /// an enumeration to identify the primitive type std::vector Params; }; struct UserDefinedType : public ParamType { /// An enumeration to identify the type id of this class. const static TypeEnum EnumTy; /// @brief Constructor. UserDefinedType(const std::string &); /// Implementation of Abstract Methods /// /// @brief Visitor service method. (see TypeVisitor for more details). /// When overridden in subclasses, preform a 'double dispatch' to the /// appropriate visit method in the given visitor. /// @param TypeVisitor type visitor. MangleError accept(TypeVisitor *) const override; /// @brief Returns a string representation of the underlying type. /// @return type as string. std::string toString() const override; /// @brief Returns true if given param type is equal to this type. /// @param ParamType given param type. /// @return true if given param type is equal to this type and false /// otherwise. bool equals(const ParamType *) const override; protected: /// The name of the user defined type. std::string Name; }; /// @brief Can be overridden so an object of static type Type* will /// dispatch the correct visit method according to its dynamic type. struct TypeVisitor { SPIRversion SpirVer; TypeVisitor(SPIRversion Ver) : SpirVer(Ver) {} virtual ~TypeVisitor() {} virtual MangleError visit(const PrimitiveType *) = 0; virtual MangleError visit(const VectorType *) = 0; virtual MangleError visit(const PointerType *) = 0; virtual MangleError visit(const AtomicType *) = 0; virtual MangleError visit(const BlockType *) = 0; virtual MangleError visit(const UserDefinedType *) = 0; }; /// @brief Template dynamic cast function for ParamType derived classes. /// @param ParamType given param type. /// @return required casting type if given param type is an instance if // that type, NULL otherwise. template T *dynCast(ParamType *PType) { assert(PType && "dyn_cast does not support casting of NULL"); return (T::EnumTy == PType->getTypeId()) ? (T *)PType : NULL; } /// @brief Template dynamic cast function for ParamType derived classes /// (the constant version). /// @param ParamType given param type. /// @return required casting type if given param type is an instance if // that type, NULL otherwise. template const T *dynCast(const ParamType *PType) { assert(PType && "dyn_cast does not support casting of NULL"); return (T::EnumTy == PType->getTypeId()) ? (const T *)PType : NULL; } } // namespace SPIR #endif // SPIRV_MANGLER_PARAMETERTYPE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/README.md000066400000000000000000000010531506646577400220270ustar00rootroot00000000000000Contributed by: Intel Corporation. SPIR Name Mangler ================= The NameMangler Library Converts the given function descriptor to a string that represents the function's prototype. The mangling algorithm is based on clang 3.0 Itanium mangling algorithm (http://sourcery.mentor.com/public/cxx-abi/abi.html#mangling). The algorithm is adapted to support mangling of SPIR built-in functions and was tested on SPIR built-ins only. The mangler supports mangling according to SPIR 1.2 and SPIR 2.0 For usage examples see unittest/spir_name_mangler. SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/Mangler/Refcount.h000066400000000000000000000036071506646577400225150ustar00rootroot00000000000000//===--------------------------- Refcount.h ------------------------------===// // // SPIR Tools // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// /* * Contributed by: Intel Corporation */ #ifndef SPIRV_MANGLER_REFCOUNT_H #define SPIRV_MANGLER_REFCOUNT_H #include namespace SPIR { template class RefCount { public: RefCount() : Count(0), Ptr(0) {} RefCount(T *Ptr) : Ptr(Ptr) { Count = new int(1); } RefCount(const RefCount &Other) { cpy(Other); } ~RefCount() { if (Count) dispose(); } RefCount &operator=(const RefCount &Other) { if (this == &Other) return *this; if (Count) dispose(); cpy(Other); return *this; } void init(T *Ptr) { assert(!Ptr && "overrunning non NULL pointer"); assert(!Count && "overrunning non NULL pointer"); Count = new int(1); this->Ptr = Ptr; } bool isNull() const { return (!Ptr); } // Pointer access const T &operator*() const { sanity(); return *Ptr; } T &operator*() { sanity(); return *Ptr; } operator T *() { return Ptr; } operator const T *() const { return Ptr; } T *operator->() { return Ptr; } const T *operator->() const { return Ptr; } private: void sanity() const { assert(Ptr && "NULL pointer"); assert(Count && "NULL ref counter"); assert(*Count && "zero ref counter"); } void cpy(const RefCount &Other) { Count = Other.Count; Ptr = Other.Ptr; if (Count) ++*Count; } void dispose() { sanity(); if (0 == --*Count) { delete Count; delete Ptr; Ptr = 0; Count = 0; } } int *Count; T *Ptr; }; // End RefCount } // namespace SPIR #endif // SPIRV_MANGLER_REFCOUNT_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLToSPIRV.cpp000066400000000000000000002457361506646577400214550ustar00rootroot00000000000000//===- OCLToSPIRV.cpp - Transform OCL to SPIR-V builtins --------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements preprocessing of OpenCL C built-in functions into SPIR-V // friendly IR form for further translation into SPIR-V // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "ocl-to-spv" #include "OCLToSPIRV.h" #include "OCLTypeToSPIRV.h" #include "SPIRVInternal.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/Debug.h" #include #include using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { static size_t getOCLCpp11AtomicMaxNumOps(StringRef Name) { return StringSwitch(Name) .Cases("load", "flag_test_and_set", "flag_clear", 3) .Cases("store", "exchange", 4) .StartsWith("compare_exchange", 6) .StartsWith("fetch", 4) .Default(0); } static Type *getBlockStructType(Value *Parameter) { // In principle, this information should be passed to us from Clang via // an elementtype attribute. However, said attribute requires that the // function call be an intrinsic, which it is not. Instead, we rely on being // able to trace this to the declaration of a variable: OpenCL C specification // section 6.12.5 should guarantee that we can do this. Value *UnderlyingObject = Parameter->stripPointerCasts(); Type *ParamType = nullptr; if (auto *GV = dyn_cast(UnderlyingObject)) ParamType = GV->getValueType(); else if (auto *Alloca = dyn_cast(UnderlyingObject)) ParamType = Alloca->getAllocatedType(); else llvm_unreachable("Blocks in OpenCL C must be traceable to allocation site"); return ParamType; } /// Return one of the SPIR-V 1.4 SignExtend or ZeroExtend image operands /// for a demangled function name, or 0 if the function does not return an /// integer type (e.g. read_imagef). static unsigned getImageSignZeroExt(StringRef DemangledName) { bool IsSigned = !DemangledName.endswith("ui") && DemangledName.back() == 'i'; bool IsUnsigned = DemangledName.endswith("ui"); if (IsSigned) return ImageOperandsMask::ImageOperandsSignExtendMask; if (IsUnsigned) return ImageOperandsMask::ImageOperandsZeroExtendMask; return 0; } bool OCLToSPIRVLegacy::runOnModule(Module &M) { setOCLTypeToSPIRV(&getAnalysis()); return runOCLToSPIRV(M); } void OCLToSPIRVLegacy::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); } llvm::PreservedAnalyses OCLToSPIRVPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { setOCLTypeToSPIRV(&MAM.getResult(M)); return runOCLToSPIRV(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } /// Get vector width from OpenCL vload* function name. SPIRVWord OCLToSPIRVBase::getVecLoadWidth(const std::string &DemangledName) { SPIRVWord Width = 0; if (DemangledName == "vloada_half") Width = 1; else { unsigned Loc = 5; if (DemangledName.find("vload_half") == 0) Loc = 10; else if (DemangledName.find("vloada_half") == 0) Loc = 11; std::stringstream SS(DemangledName.substr(Loc)); SS >> Width; } return Width; } /// Transform OpenCL vload/vstore function name. void OCLToSPIRVBase::transVecLoadStoreName(std::string &DemangledName, const std::string &Stem, bool AlwaysN) { auto HalfStem = Stem + "_half"; auto HalfStemR = HalfStem + "_r"; if (!AlwaysN && DemangledName == HalfStem) return; if (!AlwaysN && DemangledName.find(HalfStemR) == 0) { DemangledName = HalfStemR; return; } if (DemangledName.find(HalfStem) == 0) { auto OldName = DemangledName; DemangledName = HalfStem + "n"; if (OldName.find("_r") != std::string::npos) DemangledName += "_r"; return; } if (DemangledName.find(Stem) == 0) { DemangledName = Stem + "n"; return; } } char OCLToSPIRVLegacy::ID = 0; bool OCLToSPIRVBase::runOCLToSPIRV(Module &Module) { M = &Module; Ctx = &M->getContext(); auto Src = getSPIRVSource(&Module); // This is a pre-processing pass, which transform LLVM IR module to a more // suitable form for the SPIR-V translation: it is specifically designed to // handle OpenCL C built-in functions and shouldn't be launched for other // source languages if (std::get<0>(Src) != spv::SourceLanguageOpenCL_C) return false; CLVer = std::get<1>(Src); LLVM_DEBUG(dbgs() << "Enter OCLToSPIRV:\n"); visit(*M); for (auto &I : ValuesToDelete) if (auto Inst = dyn_cast(I)) Inst->eraseFromParent(); for (auto &I : ValuesToDelete) if (auto GV = dyn_cast(I)) GV->eraseFromParent(); eraseUselessFunctions(M); // remove unused functions declarations LLVM_DEBUG(dbgs() << "After OCLToSPIRV:\n" << *M); verifyRegularizationPass(*M, "OCLToSPIRV"); return true; } // The order of handling OCL builtin functions is important. // Workgroup functions need to be handled before pipe functions since // there are functions fall into both categories. void OCLToSPIRVBase::visitCallInst(CallInst &CI) { LLVM_DEBUG(dbgs() << "[visistCallInst] " << CI << '\n'); auto F = CI.getCalledFunction(); if (!F) return; auto MangledName = F->getName(); StringRef DemangledName; if (!oclIsBuiltin(MangledName, DemangledName)) return; LLVM_DEBUG(dbgs() << "DemangledName: " << DemangledName << '\n'); if (DemangledName.find(kOCLBuiltinName::NDRangePrefix) == 0) { visitCallNDRange(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::All) { visitCallAllAny(OpAll, &CI); return; } if (DemangledName == kOCLBuiltinName::Any) { visitCallAllAny(OpAny, &CI); return; } if (DemangledName.find(kOCLBuiltinName::AsyncWorkGroupCopy) == 0 || DemangledName.find(kOCLBuiltinName::AsyncWorkGroupStridedCopy) == 0) { visitCallAsyncWorkGroupCopy(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::AtomicPrefix) == 0 || DemangledName.find(kOCLBuiltinName::AtomPrefix) == 0) { // Compute atomic builtins do not support floating types. if (CI.getType()->isFloatingPointTy() && isComputeAtomicOCLBuiltin(DemangledName)) return; auto PCI = &CI; if (DemangledName == kOCLBuiltinName::AtomicInit) { visitCallAtomicInit(PCI); return; } if (DemangledName == kOCLBuiltinName::AtomicWorkItemFence) { visitCallAtomicWorkItemFence(PCI); return; } if (DemangledName == kOCLBuiltinName::AtomicCmpXchgWeak || DemangledName == kOCLBuiltinName::AtomicCmpXchgStrong || DemangledName == kOCLBuiltinName::AtomicCmpXchgWeakExplicit || DemangledName == kOCLBuiltinName::AtomicCmpXchgStrongExplicit) { assert((CLVer == kOCLVer::CL20 || CLVer == kOCLVer::CL30) && "Wrong version of OpenCL"); PCI = visitCallAtomicCmpXchg(PCI); } visitCallAtomicLegacy(PCI, MangledName, DemangledName); visitCallAtomicCpp11(PCI, MangledName, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::ConvertPrefix) == 0) { visitCallConvert(&CI, MangledName, DemangledName); return; } if (DemangledName == kOCLBuiltinName::GetImageWidth || DemangledName == kOCLBuiltinName::GetImageHeight || DemangledName == kOCLBuiltinName::GetImageDepth || DemangledName == kOCLBuiltinName::GetImageDim || DemangledName == kOCLBuiltinName::GetImageArraySize) { visitCallGetImageSize(&CI, DemangledName); return; } if ((DemangledName.find(kOCLBuiltinName::WorkGroupPrefix) == 0 && DemangledName != kOCLBuiltinName::WorkGroupBarrier) || DemangledName == kOCLBuiltinName::WaitGroupEvent || (DemangledName.find(kOCLBuiltinName::SubGroupPrefix) == 0 && DemangledName != kOCLBuiltinName::SubGroupBarrier)) { visitCallGroupBuiltin(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::MemFence || DemangledName == kOCLBuiltinName::ReadMemFence || DemangledName == kOCLBuiltinName::WriteMemFence) { visitCallMemFence(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::ReadImage) == 0) { if (MangledName.find(kMangledName::Sampler) != StringRef::npos) { visitCallReadImageWithSampler(&CI, MangledName, DemangledName); return; } if (MangledName.find("msaa") != StringRef::npos) { visitCallReadImageMSAA(&CI, MangledName); return; } } if (DemangledName.find(kOCLBuiltinName::ReadImage) == 0 || DemangledName.find(kOCLBuiltinName::WriteImage) == 0) { visitCallReadWriteImage(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::ToGlobal || DemangledName == kOCLBuiltinName::ToLocal || DemangledName == kOCLBuiltinName::ToPrivate) { visitCallToAddr(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::VLoadPrefix) == 0 || DemangledName.find(kOCLBuiltinName::VStorePrefix) == 0) { visitCallVecLoadStore(&CI, MangledName, DemangledName); return; } if (DemangledName == kOCLBuiltinName::IsFinite || DemangledName == kOCLBuiltinName::IsInf || DemangledName == kOCLBuiltinName::IsNan || DemangledName == kOCLBuiltinName::IsNormal || DemangledName == kOCLBuiltinName::Signbit) { visitCallRelational(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::WorkGroupBarrier || DemangledName == kOCLBuiltinName::Barrier || DemangledName == kOCLBuiltinName::SubGroupBarrier) { visitCallBarrier(&CI); return; } if (DemangledName == kOCLBuiltinName::GetFence) { visitCallGetFence(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::Dot && CI.getOperand(0)->getType()->isFloatingPointTy()) { visitCallDot(&CI); return; } if (DemangledName == kOCLBuiltinName::Dot || DemangledName == kOCLBuiltinName::DotAccSat || DemangledName.startswith(kOCLBuiltinName::Dot4x8PackedPrefix) || DemangledName.startswith(kOCLBuiltinName::DotAccSat4x8PackedPrefix)) { if (CI.getOperand(0)->getType()->isVectorTy()) { auto *VT = (VectorType *)(CI.getOperand(0)->getType()); if (!isa(VT->getElementType())) { visitCallBuiltinSimple(&CI, MangledName, DemangledName); return; } } visitCallDot(&CI, MangledName, DemangledName); return; } if (DemangledName.startswith(kOCLBuiltinName::ClockReadPrefix)) { visitCallClockRead(&CI, MangledName, DemangledName); return; } if (DemangledName == kOCLBuiltinName::FMin || DemangledName == kOCLBuiltinName::FMax || DemangledName == kOCLBuiltinName::Min || DemangledName == kOCLBuiltinName::Max || DemangledName == kOCLBuiltinName::Step || DemangledName == kOCLBuiltinName::SmoothStep || DemangledName == kOCLBuiltinName::Clamp || DemangledName == kOCLBuiltinName::Mix) { visitCallScalToVec(&CI, MangledName, DemangledName); return; } if (DemangledName == kOCLBuiltinName::GetImageChannelDataType) { visitCallGetImageChannel(&CI, DemangledName, OCLImageChannelDataTypeOffset); return; } if (DemangledName == kOCLBuiltinName::GetImageChannelOrder) { visitCallGetImageChannel(&CI, DemangledName, OCLImageChannelOrderOffset); return; } if (isEnqueueKernelBI(MangledName)) { visitCallEnqueueKernel(&CI, DemangledName); return; } if (isKernelQueryBI(MangledName)) { visitCallKernelQuery(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::SubgroupBlockReadINTELPrefix) == 0) { visitSubgroupBlockReadINTEL(&CI); return; } if (DemangledName.find(kOCLBuiltinName::SubgroupBlockWriteINTELPrefix) == 0) { visitSubgroupBlockWriteINTEL(&CI); return; } if (DemangledName.find(kOCLBuiltinName::SubgroupImageMediaBlockINTELPrefix) == 0) { visitSubgroupImageMediaBlockINTEL(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::SplitBarrierINTELPrefix) == 0) { visitCallSplitBarrierINTEL(&CI, DemangledName); return; } // Handle 'cl_intel_device_side_avc_motion_estimation' extension built-ins if (DemangledName.find(kOCLSubgroupsAVCIntel::Prefix) == 0 || // Workaround for a bug in the extension specification DemangledName.find("intel_sub_group_ime_ref_window_size") == 0) { if (MangledName.find(kMangledName::Sampler) != StringRef::npos) visitSubgroupAVCBuiltinCallWithSampler(&CI, DemangledName); else visitSubgroupAVCBuiltinCall(&CI, DemangledName); return; } if (DemangledName.find(kOCLBuiltinName::LDEXP) == 0) { visitCallLdexp(&CI, MangledName, DemangledName); return; } if (DemangledName == kOCLBuiltinName::ConvertBFloat16AsUShort || DemangledName == kOCLBuiltinName::ConvertBFloat162AsUShort2 || DemangledName == kOCLBuiltinName::ConvertBFloat163AsUShort3 || DemangledName == kOCLBuiltinName::ConvertBFloat164AsUShort4 || DemangledName == kOCLBuiltinName::ConvertBFloat168AsUShort8 || DemangledName == kOCLBuiltinName::ConvertBFloat1616AsUShort16) { visitCallConvertBFloat16AsUshort(&CI, DemangledName); return; } if (DemangledName == kOCLBuiltinName::ConvertAsBFloat16Float || DemangledName == kOCLBuiltinName::ConvertAsBFloat162Float2 || DemangledName == kOCLBuiltinName::ConvertAsBFloat163Float3 || DemangledName == kOCLBuiltinName::ConvertAsBFloat164Float4 || DemangledName == kOCLBuiltinName::ConvertAsBFloat168Float8 || DemangledName == kOCLBuiltinName::ConvertAsBFloat1616Float16) { visitCallConvertAsBFloat16Float(&CI, DemangledName); return; } visitCallBuiltinSimple(&CI, MangledName, DemangledName); } void OCLToSPIRVBase::visitCallNDRange(CallInst *CI, StringRef DemangledName) { assert(DemangledName.find(kOCLBuiltinName::NDRangePrefix) == 0); StringRef LenStr = DemangledName.substr(8, 1); auto Len = atoi(LenStr.data()); assert(Len >= 1 && Len <= 3); // SPIR-V ndrange structure requires 3 members in the following order: // global work offset // global work size // local work size // The arguments need to add missing members. AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { for (size_t I = 1, E = Args.size(); I != E; ++I) Args[I] = getScalarOrArray(Args[I], Len, CI); switch (Args.size()) { case 2: { // Has global work size. auto T = Args[1]->getType(); auto C = getScalarOrArrayConstantInt(CI, T, Len, 0); Args.push_back(C); Args.push_back(C); } break; case 3: { // Has global and local work size. auto T = Args[1]->getType(); Args.push_back(getScalarOrArrayConstantInt(CI, T, Len, 0)); } break; case 4: { // Move offset arg to the end auto OffsetPos = Args.begin() + 1; Value *OffsetVal = *OffsetPos; Args.erase(OffsetPos); Args.push_back(OffsetVal); } break; default: assert(0 && "Invalid number of arguments"); } // Translate ndrange_ND into differently named SPIR-V // decorated functions because they have array arugments // of different dimension which mangled the same way. std::string Postfix("_"); Postfix += LenStr; Postfix += 'D'; return getSPIRVFuncName(OpBuildNDRange, Postfix); }, &Attrs); } void OCLToSPIRVBase::visitCallAsyncWorkGroupCopy(CallInst *CI, StringRef DemangledName) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { if (DemangledName == OCLUtil::kOCLBuiltinName::AsyncWorkGroupCopy) { Args.insert(Args.begin() + 3, addSizet(1)); } Args.insert(Args.begin(), addInt32(ScopeWorkgroup)); return getSPIRVFuncName(OpGroupAsyncCopy); }, &Attrs); } CallInst *OCLToSPIRVBase::visitCallAtomicCmpXchg(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Value *Expected = nullptr; CallInst *NewCI = nullptr; mutateCallInstOCL( M, CI, [&](CallInst *CI, std::vector &Args, Type *&RetTy) { RetTy = Args[2]->getType(); if (RetTy->isFloatTy() || RetTy->isDoubleTy()) { RetTy = RetTy->isFloatTy() ? Type::getInt32Ty(*Ctx) : Type::getInt64Ty(*Ctx); Args[0] = new BitCastInst( Args[0], PointerType::get(RetTy, Args[0]->getType()->getPointerAddressSpace()), "", CI); Args[1] = new BitCastInst( Args[1], PointerType::get(RetTy, Args[1]->getType()->getPointerAddressSpace()), "", CI); Args[2] = new BitCastInst(Args[2], RetTy, "", CI); } Expected = Args[1]; // temporary save second argument. Args[1] = new LoadInst(RetTy, Args[1], "exp", false, CI); assert(Args[1]->getType()->isIntegerTy() && Args[2]->getType()->isIntegerTy() && "In SPIR-V 1.0 arguments of OpAtomicCompareExchange must be " "an integer type scalars"); return kOCLBuiltinName::AtomicCmpXchgStrong; }, [&](CallInst *NCI) -> Instruction * { NewCI = NCI; Instruction *Store = new StoreInst(NCI, Expected, NCI->getNextNode()); return new ICmpInst(Store->getNextNode(), CmpInst::ICMP_EQ, NCI, NCI->getArgOperand(1)); }, &Attrs); return NewCI; } void OCLToSPIRVBase::visitCallAtomicInit(CallInst *CI) { auto ST = new StoreInst(CI->getArgOperand(1), CI->getArgOperand(0), CI); ST->takeName(CI); CI->dropAllReferences(); CI->eraseFromParent(); } void OCLToSPIRVBase::visitCallAllAny(spv::Op OC, CallInst *CI) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); auto Args = getArguments(CI); assert(Args.size() == 1); auto *ArgTy = Args[0]->getType(); auto Zero = Constant::getNullValue(Args[0]->getType()); auto *Cmp = CmpInst::Create(CmpInst::ICmp, CmpInst::ICMP_SLT, Args[0], Zero, "cast", CI); if (!isa(ArgTy)) { auto *Cast = CastInst::CreateZExtOrBitCast(Cmp, Type::getInt32Ty(*Ctx), "", Cmp->getNextNode()); CI->replaceAllUsesWith(Cast); CI->eraseFromParent(); } else { mutateCallInstSPIRV( M, CI, [&](CallInst *, std::vector &Args, Type *&Ret) { Args[0] = Cmp; Ret = Type::getInt1Ty(*Ctx); return getSPIRVFuncName(OC); }, [&](CallInst *CI) -> Instruction * { return CastInst::CreateZExtOrBitCast(CI, Type::getInt32Ty(*Ctx), "", CI->getNextNode()); }, &Attrs); } } void OCLToSPIRVBase::visitCallAtomicWorkItemFence(CallInst *CI) { transMemoryBarrier(CI, getAtomicWorkItemFenceLiterals(CI)); } void OCLToSPIRVBase::visitCallMemFence(CallInst *CI, StringRef DemangledName) { OCLMemOrderKind MO = StringSwitch(DemangledName) .Case(kOCLBuiltinName::ReadMemFence, OCLMO_acquire) .Case(kOCLBuiltinName::WriteMemFence, OCLMO_release) .Default(OCLMO_acq_rel); // kOCLBuiltinName::MemFence transMemoryBarrier( CI, std::make_tuple(cast(CI->getArgOperand(0))->getZExtValue(), MO, OCLMS_work_group)); } void OCLToSPIRVBase::transMemoryBarrier(CallInst *CI, AtomicWorkItemFenceLiterals Lit) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(2); Args[0] = addInt32(map(std::get<2>(Lit))); Args[1] = addInt32( mapOCLMemSemanticToSPIRV(std::get<0>(Lit), std::get<1>(Lit))); return getSPIRVFuncName(OpMemoryBarrier); }, &Attrs); } void OCLToSPIRVBase::visitCallAtomicLegacy(CallInst *CI, StringRef MangledName, StringRef DemangledName) { StringRef Stem = DemangledName; if (Stem.startswith("atom_")) Stem = Stem.drop_front(strlen("atom_")); else if (Stem.startswith("atomic_")) Stem = Stem.drop_front(strlen("atomic_")); else return; std::string Sign; std::string Postfix; std::string Prefix; if (Stem == "add" || Stem == "sub" || Stem == "and" || Stem == "or" || Stem == "xor" || Stem == "min" || Stem == "max") { if ((Stem == "min" || Stem == "max") && isMangledTypeUnsigned(MangledName.back())) Sign = 'u'; Prefix = "fetch_"; Postfix = "_explicit"; } else if (Stem == "xchg") { Stem = "exchange"; Postfix = "_explicit"; } else if (Stem == "cmpxchg") { Stem = "compare_exchange_strong"; Postfix = "_explicit"; } else if (Stem == "inc" || Stem == "dec") { // do nothing } else return; OCLBuiltinTransInfo Info; Info.UniqName = "atomic_" + Prefix + Sign + Stem.str() + Postfix; std::vector PostOps; PostOps.push_back(OCLLegacyAtomicMemOrder); if (Stem.startswith("compare_exchange")) PostOps.push_back(OCLLegacyAtomicMemOrder); PostOps.push_back(OCLLegacyAtomicMemScope); Info.PostProc = [=](std::vector &Ops) { for (auto &I : PostOps) { Ops.push_back(addInt32(I)); } }; transAtomicBuiltin(CI, Info); } void OCLToSPIRVBase::visitCallAtomicCpp11(CallInst *CI, StringRef MangledName, StringRef DemangledName) { StringRef Stem = DemangledName; if (Stem.startswith("atomic_")) Stem = Stem.drop_front(strlen("atomic_")); else return; std::string NewStem(Stem); std::vector PostOps; if (Stem.startswith("store") || Stem.startswith("load") || Stem.startswith("exchange") || Stem.startswith("compare_exchange") || Stem.startswith("fetch") || Stem.startswith("flag")) { if ((Stem.startswith("fetch_min") || Stem.startswith("fetch_max")) && containsUnsignedAtomicType(MangledName)) NewStem.insert(NewStem.begin() + strlen("fetch_"), 'u'); if (!Stem.endswith("_explicit")) { NewStem = NewStem + "_explicit"; PostOps.push_back(OCLMO_seq_cst); if (Stem.startswith("compare_exchange")) PostOps.push_back(OCLMO_seq_cst); PostOps.push_back(OCLMS_device); } else { auto MaxOps = getOCLCpp11AtomicMaxNumOps(Stem.drop_back(strlen("_explicit"))); if (CI->arg_size() < MaxOps) PostOps.push_back(OCLMS_device); } } else if (Stem == "work_item_fence") { // do nothing } else return; OCLBuiltinTransInfo Info; Info.UniqName = std::string("atomic_") + NewStem; Info.PostProc = [=](std::vector &Ops) { for (auto &I : PostOps) { Ops.push_back(addInt32(I)); } }; transAtomicBuiltin(CI, Info); } void OCLToSPIRVBase::transAtomicBuiltin(CallInst *CI, OCLBuiltinTransInfo &Info) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *CI, std::vector &Args) -> std::string { Info.PostProc(Args); // Order of args in OCL20: // object, 0-2 other args, 1-2 order, scope const size_t NumOrder = getAtomicBuiltinNumMemoryOrderArgs(Info.UniqName); const size_t ArgsCount = Args.size(); const size_t ScopeIdx = ArgsCount - 1; const size_t OrderIdx = ScopeIdx - NumOrder; Args[ScopeIdx] = transOCLMemScopeIntoSPIRVScope(Args[ScopeIdx], OCLMS_device, CI); for (size_t I = 0; I < NumOrder; ++I) { Args[OrderIdx + I] = transOCLMemOrderIntoSPIRVMemorySemantics( Args[OrderIdx + I], OCLMO_seq_cst, CI); } // Order of args in SPIR-V: // object, scope, 1-2 order, 0-2 other args std::swap(Args[1], Args[ScopeIdx]); if (OrderIdx > 2) { // For atomic_compare_exchange the swap above puts Comparator/Expected // argument just where it should be, so don't move the last argument // then. int Offset = Info.UniqName.find("atomic_compare_exchange") == 0 ? 1 : 0; std::rotate(Args.begin() + 2, Args.begin() + OrderIdx, Args.end() - Offset); } llvm::Type *AtomicBuiltinsReturnType = CI->getCalledFunction()->getReturnType(); auto IsFPType = [](llvm::Type *ReturnType) { return ReturnType->isHalfTy() || ReturnType->isFloatTy() || ReturnType->isDoubleTy(); }; auto SPIRVFunctionName = getSPIRVFuncName(OCLSPIRVBuiltinMap::map(Info.UniqName)); if (!IsFPType(AtomicBuiltinsReturnType)) return SPIRVFunctionName; // Translate FP-typed atomic builtins. Currently we only need to // translate atomic_fetch_[add, sub, max, min] and atomic_fetch_[add, // sub, max, min]_explicit to related float instructions. // Translate atomic_fetch_sub to OpAtomicFAddEXT with negative value // operand auto SPIRFunctionNameForFloatAtomics = llvm::StringSwitch(SPIRVFunctionName) .Case("__spirv_AtomicIAdd", "__spirv_AtomicFAddEXT") .Case("__spirv_AtomicISub", "__spirv_AtomicFAddEXT") .Case("__spirv_AtomicSMax", "__spirv_AtomicFMaxEXT") .Case("__spirv_AtomicSMin", "__spirv_AtomicFMinEXT") .Default("others"); if (SPIRVFunctionName == "__spirv_AtomicISub") { IRBuilder<> IRB(CI); // Set float operand to its negation CI->setOperand(1, IRB.CreateFNeg(CI->getArgOperand(1))); // Update Args which is used to generate new call Args.back() = CI->getArgOperand(1); } return SPIRFunctionNameForFloatAtomics == "others" ? SPIRVFunctionName : SPIRFunctionNameForFloatAtomics; }, &Attrs); } void OCLToSPIRVBase::visitCallBarrier(CallInst *CI) { auto Lit = getBarrierLiterals(CI); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(3); // Execution scope Args[0] = addInt32(map(std::get<2>(Lit))); // Memory scope Args[1] = addInt32(map(std::get<1>(Lit))); // Use sequential consistent memory order by default. // But if the flags argument is set to 0, we use // None(Relaxed) memory order. unsigned MemFenceFlag = std::get<0>(Lit); OCLMemOrderKind MemOrder = MemFenceFlag ? OCLMO_seq_cst : OCLMO_relaxed; Args[2] = addInt32(mapOCLMemSemanticToSPIRV( MemFenceFlag, MemOrder)); // Memory semantics return getSPIRVFuncName(OpControlBarrier); }, &Attrs); } void OCLToSPIRVBase::visitCallConvert(CallInst *CI, StringRef MangledName, StringRef DemangledName) { if (eraseUselessConvert(CI, MangledName, DemangledName)) return; Op OC = OpNop; auto TargetTy = CI->getType(); auto SrcTy = CI->getArgOperand(0)->getType(); if (auto *VecTy = dyn_cast(TargetTy)) TargetTy = VecTy->getElementType(); if (auto *VecTy = dyn_cast(SrcTy)) SrcTy = VecTy->getElementType(); auto IsTargetInt = isa(TargetTy); std::string TargetTyName( DemangledName.substr(strlen(kOCLBuiltinName::ConvertPrefix))); auto FirstUnderscoreLoc = TargetTyName.find('_'); if (FirstUnderscoreLoc != std::string::npos) TargetTyName = TargetTyName.substr(0, FirstUnderscoreLoc); TargetTyName = std::string("_R") + TargetTyName; std::string Sat = DemangledName.find("_sat") != StringRef::npos ? "_sat" : ""; auto TargetSigned = DemangledName[8] != 'u'; if (isa(SrcTy)) { bool Signed = isLastFuncParamSigned(MangledName); if (IsTargetInt) { if (!Sat.empty() && TargetSigned != Signed) { OC = Signed ? OpSatConvertSToU : OpSatConvertUToS; Sat = ""; } else OC = Signed ? OpSConvert : OpUConvert; } else OC = Signed ? OpConvertSToF : OpConvertUToF; } else { if (IsTargetInt) { OC = TargetSigned ? OpConvertFToS : OpConvertFToU; } else OC = OpFConvert; } auto Loc = DemangledName.find("_rt"); std::string Rounding; if (Loc != StringRef::npos && !(isa(SrcTy) && IsTargetInt)) { Rounding = DemangledName.substr(Loc, 4).str(); } assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { return getSPIRVFuncName(OC, TargetTyName + Sat + Rounding); }, &Attrs); } void OCLToSPIRVBase::visitCallGroupBuiltin(CallInst *CI, StringRef OrigDemangledName) { auto F = CI->getCalledFunction(); std::vector PreOps; std::string DemangledName{OrigDemangledName}; if (DemangledName == kOCLBuiltinName::WorkGroupBarrier) return; if (DemangledName == kOCLBuiltinName::WaitGroupEvent) { PreOps.push_back(ScopeWorkgroup); } else if (DemangledName.find(kOCLBuiltinName::WorkGroupPrefix) == 0) { DemangledName.erase(0, strlen(kOCLBuiltinName::WorkPrefix)); PreOps.push_back(ScopeWorkgroup); } else if (DemangledName.find(kOCLBuiltinName::SubGroupPrefix) == 0) { DemangledName.erase(0, strlen(kOCLBuiltinName::SubPrefix)); PreOps.push_back(ScopeSubgroup); } else return; if (DemangledName != kOCLBuiltinName::WaitGroupEvent) { StringRef FuncName = DemangledName; FuncName = FuncName.drop_front(strlen(kSPIRVName::GroupPrefix)); SPIRSPIRVGroupOperationMap::foreachConditional( [&](const std::string &S, SPIRVGroupOperationKind G) { if (!FuncName.startswith(S)) return true; // continue PreOps.push_back(G); StringRef Op = StringSwitch(FuncName) .StartsWith("ballot", "group_ballot_bit_count_") .StartsWith("non_uniform", kSPIRVName::GroupNonUniformPrefix) .Default(kSPIRVName::GroupPrefix); // clustered functions are handled with non uniform group opcodes StringRef ClusteredOp = FuncName.contains("clustered_") ? "non_uniform_" : ""; StringRef LogicalOp = FuncName.contains("logical_") ? "logical_" : ""; StringRef GroupOp = StringSwitch(FuncName) .Case("ballot_bit_count", "add") .Case("ballot_inclusive_scan", "add") .Case("ballot_exclusive_scan", "add") .Default(FuncName.take_back( 3)); // assumes op is three characters GroupOp.consume_front("_"); // when op is two characters assert(!GroupOp.empty() && "Invalid OpenCL group builtin function"); char OpTyC = 0; auto OpTy = F->getReturnType(); if (OpTy->isFloatingPointTy()) OpTyC = 'f'; else if (OpTy->isIntegerTy()) { auto NeedSign = GroupOp == "max" || GroupOp == "min"; if (!NeedSign) OpTyC = 'i'; else { // clustered reduce args are (type, uint) // other operation args are (type) auto MangledName = F->getName(); auto MangledTyC = ClusteredOp.empty() ? MangledName.back() : MangledName.take_back(2).front(); if (isMangledTypeSigned(MangledTyC)) OpTyC = 's'; else OpTyC = 'u'; } } else llvm_unreachable("Invalid OpenCL group builtin argument type"); DemangledName = Op.str() + ClusteredOp.str() + LogicalOp.str() + OpTyC + GroupOp.str(); return false; // break out of loop }); } const bool IsElect = DemangledName == "group_elect"; const bool IsAllOrAny = (DemangledName.find("_all") != std::string::npos || DemangledName.find("_any") != std::string::npos); const bool IsAllEqual = DemangledName.find("_all_equal") != std::string::npos; const bool IsBallot = DemangledName == "group_ballot"; const bool IsInverseBallot = DemangledName == "group_inverse_ballot"; const bool IsBallotBitExtract = DemangledName == "group_ballot_bit_extract"; const bool IsLogical = DemangledName.find("_logical") != std::string::npos; const bool HasBoolReturnType = IsElect || IsAllOrAny || IsAllEqual || IsInverseBallot || IsBallotBitExtract || IsLogical; const bool HasBoolArg = (IsAllOrAny && !IsAllEqual) || IsBallot || IsLogical; auto Consts = getInt32(M, PreOps); OCLBuiltinTransInfo Info; if (HasBoolReturnType) Info.RetTy = Type::getInt1Ty(*Ctx); Info.UniqName = DemangledName; Info.PostProc = [=](std::vector &Ops) { if (HasBoolArg) { IRBuilder<> IRB(CI); Ops[0] = IRB.CreateICmpNE(Ops[0], ConstantInt::get(Type::getInt32Ty(*Ctx), 0)); } size_t E = Ops.size(); if (DemangledName == "group_broadcast" && E > 2) { assert(E == 3 || E == 4); makeVector(CI, Ops, std::make_pair(Ops.begin() + 1, Ops.end())); } Ops.insert(Ops.begin(), Consts.begin(), Consts.end()); }; transBuiltin(CI, Info); } void OCLToSPIRVBase::transBuiltin(CallInst *CI, OCLBuiltinTransInfo &Info) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Op OC = OpNop; unsigned ExtOp = ~0U; SPIRVBuiltinVariableKind BVKind = BuiltInMax; if (StringRef(Info.UniqName).startswith(kSPIRVName::Prefix)) return; if (OCLSPIRVBuiltinMap::find(Info.UniqName, &OC)) { if (OC == OpImageRead) { // There are several read_image* functions defined by OpenCL C spec, but // all of them use the same SPIR-V Instruction - some of them might only // differ by return type, so, we need to include return type into the // mangling scheme to get them differentiated. // // Example: int4 read_imagei(image2d_t, sampler_t, int2) // uint4 read_imageui(image2d_t, sampler_t, int2) // Both functions above are represented by the same SPIR-V // instruction: argument types are the same, only return type is // different Info.UniqName = getSPIRVFuncName(OC, CI->getType()); } else { Info.UniqName = getSPIRVFuncName(OC); } } else if ((ExtOp = getExtOp(Info.MangledName, Info.UniqName)) != ~0U) Info.UniqName = getSPIRVExtFuncName(SPIRVEIS_OpenCL, ExtOp); else if (SPIRSPIRVBuiltinVariableMap::find(Info.UniqName, &BVKind)) { // Map OCL work item builtins to SPV-IR work item builtins. // e.g. get_global_id() --> __spirv_BuiltinGlobalInvocationId() Info.UniqName = getSPIRVFuncName(BVKind); } else return; if (!Info.RetTy) mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Info.PostProc(Args); return Info.UniqName + Info.Postfix; }, &Attrs); else mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { Info.PostProc(Args); RetTy = Info.RetTy; return Info.UniqName + Info.Postfix; }, [=](CallInst *NewCI) -> Instruction * { if (NewCI->getType()->isIntegerTy() && CI->getType()->isIntegerTy()) return CastInst::CreateIntegerCast(NewCI, CI->getType(), Info.IsRetSigned, "", CI); else return CastInst::CreatePointerBitCastOrAddrSpaceCast( NewCI, CI->getType(), "", CI); }, &Attrs); } void OCLToSPIRVBase::visitCallReadImageMSAA(CallInst *CI, StringRef MangledName) { assert(MangledName.find("msaa") != StringRef::npos); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Args.insert(Args.begin() + 2, getInt32(M, ImageOperandsSampleMask)); return getSPIRVFuncName(OpImageRead, std::string(kSPIRVPostfix::ExtDivider) + getPostfixForReturnType(CI)); }, &Attrs); } void OCLToSPIRVBase::visitCallReadImageWithSampler(CallInst *CI, StringRef MangledName, StringRef DemangledName) { assert(MangledName.find(kMangledName::Sampler) != StringRef::npos); assert(CI->getCalledFunction() && "Unexpected indirect call"); Function *Func = CI->getCalledFunction(); AttributeList Attrs = Func->getAttributes(); bool IsRetScalar = !CI->getType()->isVectorTy(); SmallVector ArgStructTys; getParameterTypes(CI, ArgStructTys); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&Ret) { auto *ImageTy = OCLTypeToSPIRVPtr->getAdaptedArgumentType(Func, 0).second; if (!ImageTy) ImageTy = ArgStructTys[0]; ImageTy = adaptSPIRVImageType(M, ImageTy); auto SampledImgTy = getSPIRVTypeByChangeBaseTypeName( M, ImageTy, kSPIRVTypeName::Image, kSPIRVTypeName::SampledImg); Value *SampledImgArgs[] = {Args[0], Args[1]}; auto SampledImg = addCallInstSPIRV( M, getSPIRVFuncName(OpSampledImage), SampledImgTy, SampledImgArgs, nullptr, {ArgStructTys[0], ArgStructTys[1]}, CI, kSPIRVName::TempSampledImage); Args[0] = SampledImg; Args.erase(Args.begin() + 1, Args.begin() + 2); unsigned ImgOpMask = getImageSignZeroExt(DemangledName); unsigned ImgOpMaskInsIndex = Args.size(); switch (Args.size()) { case 2: // no lod ImgOpMask |= ImageOperandsMask::ImageOperandsLodMask; ImgOpMaskInsIndex = Args.size(); Args.push_back(getFloat32(M, 0.f)); break; case 3: // explicit lod ImgOpMask |= ImageOperandsMask::ImageOperandsLodMask; ImgOpMaskInsIndex = 2; break; case 4: // gradient ImgOpMask |= ImageOperandsMask::ImageOperandsGradMask; ImgOpMaskInsIndex = 2; break; default: assert(0 && "read_image* with unhandled number of args!"); } Args.insert(Args.begin() + ImgOpMaskInsIndex, getInt32(M, ImgOpMask)); // SPIR-V instruction always returns 4-element vector if (IsRetScalar) Ret = FixedVectorType::get(Ret, 4); return getSPIRVFuncName(OpImageSampleExplicitLod, std::string(kSPIRVPostfix::ExtDivider) + getPostfixForReturnType(Ret)); }, [&](CallInst *CI) -> Instruction * { if (IsRetScalar) return ExtractElementInst::Create(CI, getSizet(M, 0), "", CI->getNextNode()); return CI; }, &Attrs); } void OCLToSPIRVBase::visitCallGetImageSize(CallInst *CI, StringRef DemangledName) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); StringRef TyName; SmallVector SubStrs; SmallVector ParamTys; getParameterTypes(CI, ParamTys); auto IsImg = isOCLImageStructType(ParamTys[0], &TyName); (void)IsImg; assert(IsImg); std::string ImageTyName = getImageBaseTypeName(TyName); auto Desc = map(ImageTyName); unsigned Dim = getImageDimension(Desc.Dim) + Desc.Arrayed; assert(Dim > 0 && "Invalid image dimension."); mutateCallInstSPIRV( M, CI, [&](CallInst *, std::vector &Args, Type *&Ret) { assert(Args.size() == 1); Ret = CI->getType()->isIntegerTy(64) ? Type::getInt64Ty(*Ctx) : Type::getInt32Ty(*Ctx); if (Dim > 1) Ret = FixedVectorType::get(Ret, Dim); if (Desc.Dim == DimBuffer) return getSPIRVFuncName(OpImageQuerySize, CI->getType()); else { Args.push_back(getInt32(M, 0)); return getSPIRVFuncName(OpImageQuerySizeLod, CI->getType()); } }, [&](CallInst *NCI) -> Instruction * { if (Dim == 1) return NCI; if (DemangledName == kOCLBuiltinName::GetImageDim) { if (Desc.Dim == Dim3D) { auto ZeroVec = ConstantVector::getSplat( ElementCount::getFixed(3), Constant::getNullValue( cast(NCI->getType())->getElementType())); Constant *Index[] = {getInt32(M, 0), getInt32(M, 1), getInt32(M, 2), getInt32(M, 3)}; return new ShuffleVectorInst(NCI, ZeroVec, ConstantVector::get(Index), "", CI); } else if (Desc.Dim == Dim2D && Desc.Arrayed) { Constant *Index[] = {getInt32(M, 0), getInt32(M, 1)}; Constant *Mask = ConstantVector::get(Index); return new ShuffleVectorInst(NCI, UndefValue::get(NCI->getType()), Mask, NCI->getName(), CI); } return NCI; } unsigned I = StringSwitch(DemangledName) .Case(kOCLBuiltinName::GetImageWidth, 0) .Case(kOCLBuiltinName::GetImageHeight, 1) .Case(kOCLBuiltinName::GetImageDepth, 2) .Case(kOCLBuiltinName::GetImageArraySize, Dim - 1); return ExtractElementInst::Create(NCI, getUInt32(M, I), "", NCI->getNextNode()); }, &Attrs); } /// Remove trivial conversion functions bool OCLToSPIRVBase::eraseUselessConvert(CallInst *CI, StringRef MangledName, StringRef DemangledName) { auto TargetTy = CI->getType(); auto SrcTy = CI->getArgOperand(0)->getType(); if (auto *VecTy = dyn_cast(TargetTy)) TargetTy = VecTy->getElementType(); if (auto *VecTy = dyn_cast(SrcTy)) SrcTy = VecTy->getElementType(); if (TargetTy == SrcTy) { if (isa(TargetTy) && DemangledName.find("_sat") != StringRef::npos && isLastFuncParamSigned(MangledName) != (DemangledName[8] != 'u')) return false; CI->getArgOperand(0)->takeName(CI); SPIRVDBG(dbgs() << "[regularizeOCLConvert] " << *CI << " <- " << *CI->getArgOperand(0) << '\n'); CI->replaceAllUsesWith(CI->getArgOperand(0)); ValuesToDelete.insert(CI); ValuesToDelete.insert(CI->getCalledFunction()); return true; } return false; } void OCLToSPIRVBase::visitCallBuiltinSimple(CallInst *CI, StringRef MangledName, StringRef DemangledName) { OCLBuiltinTransInfo Info; Info.MangledName = MangledName.str(); Info.UniqName = DemangledName.str(); transBuiltin(CI, Info); } void OCLToSPIRVBase::visitCallReadWriteImage(CallInst *CI, StringRef DemangledName) { OCLBuiltinTransInfo Info; if (DemangledName.find(kOCLBuiltinName::ReadImage) == 0) { Info.UniqName = kOCLBuiltinName::ReadImage; unsigned ImgOpMask = getImageSignZeroExt(DemangledName); if (ImgOpMask) { Info.PostProc = [&](std::vector &Args) { Args.push_back(getInt32(M, ImgOpMask)); }; } } if (DemangledName.find(kOCLBuiltinName::WriteImage) == 0) { Info.UniqName = kOCLBuiltinName::WriteImage; Info.PostProc = [&](std::vector &Args) { unsigned ImgOpMask = getImageSignZeroExt(DemangledName); unsigned ImgOpMaskInsIndex = Args.size(); if (Args.size() == 4) // write with lod { auto Lod = Args[2]; Args.erase(Args.begin() + 2); ImgOpMask |= ImageOperandsMask::ImageOperandsLodMask; ImgOpMaskInsIndex = Args.size(); Args.push_back(Lod); } if (ImgOpMask) { Args.insert(Args.begin() + ImgOpMaskInsIndex, getInt32(M, ImgOpMask)); } }; } transBuiltin(CI, Info); } void OCLToSPIRVBase::visitCallToAddr(CallInst *CI, StringRef DemangledName) { auto AddrSpace = static_cast(CI->getType()->getPointerAddressSpace()); OCLBuiltinTransInfo Info; Info.UniqName = DemangledName.str(); Info.Postfix = std::string(kSPIRVPostfix::Divider) + "To" + SPIRAddrSpaceCapitalizedNameMap::map(AddrSpace); auto StorageClass = addInt32(SPIRSPIRVAddrSpaceMap::map(AddrSpace)); Info.RetTy = getInt8PtrTy(cast(CI->getType())); Info.PostProc = [=](std::vector &Ops) { auto P = Ops.back(); Ops.pop_back(); Ops.push_back(castToInt8Ptr(P, CI)); Ops.push_back(StorageClass); }; transBuiltin(CI, Info); } void OCLToSPIRVBase::visitCallRelational(CallInst *CI, StringRef DemangledName) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Op OC = OpNop; OCLSPIRVBuiltinMap::find(DemangledName.str(), &OC); std::string SPIRVName = getSPIRVFuncName(OC); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&Ret) { Ret = Type::getInt1Ty(*Ctx); if (CI->getOperand(0)->getType()->isVectorTy()) Ret = FixedVectorType::get( Type::getInt1Ty(*Ctx), cast(CI->getOperand(0)->getType()) ->getNumElements()); return SPIRVName; }, [=](CallInst *NewCI) -> Instruction * { Value *False = nullptr, *True = nullptr; if (NewCI->getType()->isVectorTy()) { Type *IntTy = Type::getInt32Ty(*Ctx); if (cast(NewCI->getOperand(0)->getType()) ->getElementType() ->isDoubleTy()) IntTy = Type::getInt64Ty(*Ctx); if (cast(NewCI->getOperand(0)->getType()) ->getElementType() ->isHalfTy()) IntTy = Type::getInt16Ty(*Ctx); Type *VTy = FixedVectorType::get( IntTy, cast(NewCI->getType())->getNumElements()); False = Constant::getNullValue(VTy); True = Constant::getAllOnesValue(VTy); } else { False = getInt32(M, 0); True = getInt32(M, 1); } return SelectInst::Create(NewCI, True, False, "", NewCI->getNextNode()); }, &Attrs); } void OCLToSPIRVBase::visitCallVecLoadStore(CallInst *CI, StringRef MangledName, StringRef OrigDemangledName) { std::vector PreOps; std::string DemangledName{OrigDemangledName}; if (DemangledName.find(kOCLBuiltinName::VLoadPrefix) == 0 && DemangledName != kOCLBuiltinName::VLoadHalf) { SPIRVWord Width = getVecLoadWidth(DemangledName); SPIRVDBG(spvdbgs() << "[visitCallVecLoadStore] DemangledName: " << DemangledName << " Width: " << Width << '\n'); PreOps.push_back(Width); } else if (DemangledName.find(kOCLBuiltinName::RoundingPrefix) != std::string::npos) { auto R = SPIRSPIRVFPRoundingModeMap::map(DemangledName.substr( DemangledName.find(kOCLBuiltinName::RoundingPrefix) + 1, 3)); PreOps.push_back(R); } if (DemangledName.find(kOCLBuiltinName::VLoadAPrefix) == 0) transVecLoadStoreName(DemangledName, kOCLBuiltinName::VLoadAPrefix, true); else transVecLoadStoreName(DemangledName, kOCLBuiltinName::VLoadPrefix, false); if (DemangledName.find(kOCLBuiltinName::VStoreAPrefix) == 0) transVecLoadStoreName(DemangledName, kOCLBuiltinName::VStoreAPrefix, true); else transVecLoadStoreName(DemangledName, kOCLBuiltinName::VStorePrefix, false); auto Consts = getInt32(M, PreOps); OCLBuiltinTransInfo Info; Info.MangledName = MangledName.str(); Info.UniqName = DemangledName; if (DemangledName.find(kOCLBuiltinName::VLoadPrefix) == 0) Info.Postfix = std::string(kSPIRVPostfix::ExtDivider) + getPostfixForReturnType(CI); Info.PostProc = [=](std::vector &Ops) { Ops.insert(Ops.end(), Consts.begin(), Consts.end()); }; transBuiltin(CI, Info); } void OCLToSPIRVBase::visitCallGetFence(CallInst *CI, StringRef DemangledName) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Op OC = OpNop; OCLSPIRVBuiltinMap::find(DemangledName.str(), &OC); std::string SPIRVName = getSPIRVFuncName(OC); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&Ret) { return SPIRVName; }, [=](CallInst *NewCI) -> Instruction * { return BinaryOperator::CreateLShr(NewCI, getInt32(M, 8), "", CI); }, &Attrs); } void OCLToSPIRVBase::visitCallDot(CallInst *CI) { IRBuilder<> Builder(CI); Value *FMulVal = Builder.CreateFMul(CI->getOperand(0), CI->getOperand(1)); CI->replaceAllUsesWith(FMulVal); CI->eraseFromParent(); } void OCLToSPIRVBase::visitCallDot(CallInst *CI, StringRef MangledName, StringRef DemangledName) { // translation for dot function calls, // to differentiate between integer dot products bool IsFirstSigned, IsSecondSigned; bool IsDot = DemangledName == kOCLBuiltinName::Dot; bool IsAccSat = DemangledName.contains(kOCLBuiltinName::DotAccSat); bool IsPacked = CI->getOperand(0)->getType()->isIntegerTy(); if (!IsPacked) { if (IsDot) { // dot(char4, char4) _Z3dotDv4_cS_ // dot(char4, uchar4) _Z3dotDv4_cDv4_h // dot(uchar4, char4) _Z3dotDv4_hDv4_c // dot(uchar4, uchar4) _Z3dotDv4_hS_ // or // dot(short2, short2) _Z3dotDv2_sS_ // dot(short2, ushort2) _Z3dotDv2_sDv2_t // dot(ushort2, short2) _Z3dotDv2_tDv2_s // dot(ushort2, ushort2) _Z3dotDv2_tS_ assert(MangledName.startswith("_Z3dotDv")); if (MangledName[MangledName.size() - 1] == '_') { IsFirstSigned = ((MangledName[MangledName.size() - 3] == 'c') || (MangledName[MangledName.size() - 3] == 's')); IsSecondSigned = IsFirstSigned; } else { IsFirstSigned = ((MangledName[MangledName.size() - 6] == 'c') || (MangledName[MangledName.size() - 6] == 's')); IsSecondSigned = ((MangledName[MangledName.size() - 1] == 'c') || (MangledName[MangledName.size() - 1] == 's')); } } else { // dot_acc_sat(char4, char4, int) _Z11dot_acc_satDv4_cS_i // dot_acc_sat(char4, uchar4, int) _Z11dot_acc_satDv4_cDv4_hi // dot_acc_sat(uchar4, char4, int) _Z11dot_acc_satDv4_hDv4_ci // dot_acc_sat(uchar4, uchar4, uint) _Z11dot_acc_satDv4_hS_j // or // dot_acc_sat(short2, short2, int) _Z11dot_acc_satDv4_sS_i // dot_acc_sat(short2, ushort2, int) _Z11dot_acc_satDv4_sDv4_ti // dot_acc_sat(ushort2, short2, int) _Z11dot_acc_satDv4_tDv4_si // dot_acc_sat(ushort2, ushort2, uint) _Z11dot_acc_satDv4_tS_j assert(MangledName.startswith("_Z11dot_acc_satDv")); IsFirstSigned = ((MangledName[19] == 'c') || (MangledName[19] == 's')); IsSecondSigned = (MangledName[20] == 'S' ? IsFirstSigned : ((MangledName[MangledName.size() - 2] == 'c') || (MangledName[MangledName.size() - 2] == 's'))); } } else { // for packed format // dot_4x8packed_ss_int(uint, uint) _Z20dot_4x8packed_ss_intjj // dot_4x8packed_su_int(uint, uint) _Z20dot_4x8packed_su_intjj // dot_4x8packed_us_int(uint, uint) _Z20dot_4x8packed_us_intjj // dot_4x8packed_uu_uint(uint, uint) _Z21dot_4x8packed_uu_uintjj // or // dot_acc_sat_4x8packed_ss_int(uint, uint, int) // _Z28dot_acc_sat_4x8packed_ss_intjji // dot_acc_sat_4x8packed_su_int(uint, uint, int) // _Z28dot_acc_sat_4x8packed_su_intjji // dot_acc_sat_4x8packed_us_int(uint, uint, int) // _Z28dot_acc_sat_4x8packed_us_intjji // dot_acc_sat_4x8packed_uu_uint(uint, uint, uint) // _Z29dot_acc_sat_4x8packed_uu_uintjjj assert(MangledName.startswith("_Z20dot_4x8packed") || MangledName.startswith("_Z21dot_4x8packed") || MangledName.startswith("_Z28dot_acc_sat_4x8packed") || MangledName.startswith("_Z29dot_acc_sat_4x8packed")); size_t SignIndex = IsAccSat ? strlen(kOCLBuiltinName::DotAccSat4x8PackedPrefix) : strlen(kOCLBuiltinName::Dot4x8PackedPrefix); IsFirstSigned = DemangledName[SignIndex] == 's'; IsSecondSigned = DemangledName[SignIndex + 1] == 's'; } AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { // If arguments are in order unsigned -> signed // then the translator should swap them, // so that the OpSUDotKHR can be used properly if (IsFirstSigned == false && IsSecondSigned == true) { std::swap(Args[0], Args[1]); } Op OC; if (!IsAccSat) { OC = (IsFirstSigned != IsSecondSigned ? OpSUDot : ((IsFirstSigned) ? OpSDot : OpUDot)); } else { OC = (IsFirstSigned != IsSecondSigned ? OpSUDotAccSat : ((IsFirstSigned) ? OpSDotAccSat : OpUDotAccSat)); } if (IsPacked) { // As per SPIRV specification the dot OpCodes // which use scalar integers to represent // packed vectors need additional argument // specified - the Packed Vector Format Args.push_back( getInt32(M, PackedVectorFormatPackedVectorFormat4x8BitKHR)); } return getSPIRVFuncName(OC); }, &Attrs); } void OCLToSPIRVBase::visitCallClockRead(CallInst *CI, StringRef MangledName, StringRef DemangledName) { // The builtin returns i64 or <2 x i32>, but both variants are mapped to the // same instruction; hence include the return type. const std::string OpName = getSPIRVFuncName(OpReadClockKHR, CI->getType()); // Scope is part of the OpenCL builtin name. const Scope ScopeArg = StringSwitch(DemangledName) .EndsWith("device", ScopeDevice) .EndsWith("work_group", ScopeWorkgroup) .EndsWith("sub_group", ScopeSubgroup) .Default(ScopeMax); mutateCallInstSPIRV(M, CI, [=](CallInst *, std::vector &Args) { Args.push_back(getInt32(M, ScopeArg)); return OpName; }); } void OCLToSPIRVBase::visitCallScalToVec(CallInst *CI, StringRef MangledName, StringRef DemangledName) { // Check if all arguments have the same type - it's simple case. auto Uniform = true; auto IsArg0Vector = isa(CI->getOperand(0)->getType()); for (unsigned I = 1, E = CI->arg_size(); Uniform && (I != E); ++I) { Uniform = isa(CI->getOperand(I)->getType()) == IsArg0Vector; } if (Uniform) { visitCallBuiltinSimple(CI, MangledName, DemangledName); return; } std::vector VecPos; std::vector ScalarPos; if (DemangledName == kOCLBuiltinName::FMin || DemangledName == kOCLBuiltinName::FMax || DemangledName == kOCLBuiltinName::Min || DemangledName == kOCLBuiltinName::Max) { VecPos.push_back(0); ScalarPos.push_back(1); } else if (DemangledName == kOCLBuiltinName::Clamp) { VecPos.push_back(0); ScalarPos.push_back(1); ScalarPos.push_back(2); } else if (DemangledName == kOCLBuiltinName::Mix) { VecPos.push_back(0); VecPos.push_back(1); ScalarPos.push_back(2); } else if (DemangledName == kOCLBuiltinName::Step) { VecPos.push_back(1); ScalarPos.push_back(0); } else if (DemangledName == kOCLBuiltinName::SmoothStep) { VecPos.push_back(2); ScalarPos.push_back(0); ScalarPos.push_back(1); } AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(VecPos.size() + ScalarPos.size()); for (auto I : VecPos) { Args[I] = CI->getOperand(I); } auto VecElemCount = cast(CI->getOperand(VecPos[0])->getType()) ->getElementCount(); for (auto I : ScalarPos) { Instruction *Inst = InsertElementInst::Create( UndefValue::get(CI->getOperand(VecPos[0])->getType()), CI->getOperand(I), getInt32(M, 0), "", CI); Value *NewVec = new ShuffleVectorInst( Inst, UndefValue::get(CI->getOperand(VecPos[0])->getType()), ConstantVector::getSplat(VecElemCount, getInt32(M, 0)), "", CI); Args[I] = NewVec; } return getSPIRVExtFuncName(SPIRVEIS_OpenCL, getExtOp(MangledName, DemangledName)); }, &Attrs); } void OCLToSPIRVBase::visitCallGetImageChannel(CallInst *CI, StringRef DemangledName, unsigned int Offset) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Op OC = OpNop; OCLSPIRVBuiltinMap::find(DemangledName.str(), &OC); std::string SPIRVName = getSPIRVFuncName(OC); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&Ret) { return SPIRVName; }, [=](CallInst *NewCI) -> Instruction * { return BinaryOperator::CreateAdd(NewCI, getInt32(M, Offset), "", CI); }, &Attrs); } void OCLToSPIRVBase::visitCallEnqueueKernel(CallInst *CI, StringRef DemangledName) { const DataLayout &DL = M->getDataLayout(); bool HasEvents = DemangledName.find("events") != StringRef::npos; // SPIRV OpEnqueueKernel instruction has 10+ arguments. SmallVector Args; // Copy all arguments before block invoke function pointer // which match with what Clang 6.0 produced const unsigned BlockFIdx = HasEvents ? 6 : 3; Args.assign(CI->arg_begin(), CI->arg_begin() + BlockFIdx); // If no event arguments in original call, add dummy ones if (!HasEvents) { Args.push_back(getInt32(M, 0)); // dummy num events Args.push_back(getOCLNullClkEventPtr(M)); // dummy wait events Args.push_back(getOCLNullClkEventPtr(M)); // dummy ret event } // Invoke: Pointer to invoke function Value *BlockFunc = CI->getArgOperand(BlockFIdx); Args.push_back(cast(getUnderlyingObject(BlockFunc))); // Param: Pointer to block literal Value *BlockLiteral = CI->getArgOperand(BlockFIdx + 1); Args.push_back(BlockLiteral); // Param Size: Size of block literal structure // Param Aligment: Aligment of block literal structure // TODO: these numbers should be obtained from block literal structure Type *ParamType = getBlockStructType(BlockLiteral); Args.push_back(getInt32(M, DL.getTypeStoreSize(ParamType))); Args.push_back(getInt32(M, DL.getPrefTypeAlignment(ParamType))); // Local sizes arguments: Sizes of block invoke arguments // Clang 6.0 and higher generates local size operands as an array, // so we need to unpack them if (DemangledName.find("_varargs") != StringRef::npos) { const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6; auto *LocalSizeArray = cast(CI->getArgOperand(LocalSizeArrayIdx)); auto *LocalSizeArrayTy = cast(LocalSizeArray->getSourceElementType()); const uint64_t LocalSizeNum = LocalSizeArrayTy->getNumElements(); for (unsigned I = 0; I < LocalSizeNum; ++I) Args.push_back(GetElementPtrInst::Create( LocalSizeArray->getSourceElementType(), // Pointee type LocalSizeArray->getPointerOperand(), // Alloca {getInt32(M, 0), getInt32(M, I)}, // Indices "", CI)); } StringRef NewName = "__spirv_EnqueueKernel__"; FunctionType *FT = FunctionType::get(CI->getType(), getTypes(Args), false /*isVarArg*/); Function *NewF = Function::Create(FT, GlobalValue::ExternalLinkage, NewName, M); NewF->setCallingConv(CallingConv::SPIR_FUNC); CallInst *NewCall = CallInst::Create(NewF, Args, "", CI); NewCall->setCallingConv(NewF->getCallingConv()); CI->replaceAllUsesWith(NewCall); CI->eraseFromParent(); } void OCLToSPIRVBase::visitCallKernelQuery(CallInst *CI, StringRef DemangledName) { const DataLayout &DL = M->getDataLayout(); bool HasNDRange = DemangledName.find("_for_ndrange_impl") != StringRef::npos; // BIs with "_for_ndrange_impl" suffix has NDRange argument first, and // Invoke argument following. For other BIs Invoke function is the first arg const unsigned BlockFIdx = HasNDRange ? 1 : 0; Value *BlockFVal = CI->getArgOperand(BlockFIdx)->stripPointerCasts(); auto *BlockF = cast(getUnderlyingObject(BlockFVal)); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInst( M, CI, [=](CallInst *CI, std::vector &Args) { Value *Param = *Args.rbegin(); Type *ParamType = getBlockStructType(Param); // Last arg corresponds to SPIRV Param operand. // Insert Invoke in front of Param. // Add Param Size and Param Align at the end. Args[BlockFIdx] = BlockF; Args.push_back(getInt32(M, DL.getTypeStoreSize(ParamType))); Args.push_back(getInt32(M, DL.getPrefTypeAlignment(ParamType))); Op Opcode = OCLSPIRVBuiltinMap::map(DemangledName.str()); // Adding "__" postfix, so in case we have multiple such // functions and their names will have numerical postfix, // then the numerical postfix will be droped and we will get // correct function name. return getSPIRVFuncName(Opcode, kSPIRVName::Postfix); }, /*BuiltinFuncMangleInfo*/ nullptr, &Attrs); } // Add postfix to overloaded intel subgroup block read/write builtins // so new functions can be distinguished. static void processSubgroupBlockReadWriteINTEL(CallInst *CI, OCLBuiltinTransInfo &Info, const Type *DataTy, Module *M) { unsigned VectorNumElements = 1; if (auto *VecTy = dyn_cast(DataTy)) VectorNumElements = VecTy->getNumElements(); unsigned ElementBitSize = DataTy->getScalarSizeInBits(); Info.Postfix = "_"; Info.Postfix += getIntelSubgroupBlockDataPostfix(ElementBitSize, VectorNumElements); assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [&Info](CallInst *, std::vector &Args) { Info.PostProc(Args); return Info.UniqName + Info.Postfix; }, &Attrs); } // The intel_sub_group_block_read built-ins are overloaded to support both // buffers and images, but need to be mapped to distinct SPIR-V instructions. // Additionally, for block reads, need to distinguish between scalar block // reads and vector block reads. void OCLToSPIRVBase::visitSubgroupBlockReadINTEL(CallInst *CI) { OCLBuiltinTransInfo Info; SmallVector ParamTys; getParameterTypes(CI, ParamTys); if (isOCLImageStructType(ParamTys[0])) Info.UniqName = getSPIRVFuncName(spv::OpSubgroupImageBlockReadINTEL); else Info.UniqName = getSPIRVFuncName(spv::OpSubgroupBlockReadINTEL); Type *DataTy = CI->getType(); processSubgroupBlockReadWriteINTEL(CI, Info, DataTy, M); } // The intel_sub_group_block_write built-ins are similarly overloaded to support // both buffers and images but need to be mapped to distinct SPIR-V // instructions. void OCLToSPIRVBase::visitSubgroupBlockWriteINTEL(CallInst *CI) { OCLBuiltinTransInfo Info; SmallVector ParamTys; getParameterTypes(CI, ParamTys); if (isOCLImageStructType(ParamTys[0])) Info.UniqName = getSPIRVFuncName(spv::OpSubgroupImageBlockWriteINTEL); else Info.UniqName = getSPIRVFuncName(spv::OpSubgroupBlockWriteINTEL); assert(!CI->arg_empty() && "Intel subgroup block write should have arguments"); unsigned DataArg = CI->arg_size() - 1; Type *DataTy = CI->getArgOperand(DataArg)->getType(); processSubgroupBlockReadWriteINTEL(CI, Info, DataTy, M); } void OCLToSPIRVBase::visitSubgroupImageMediaBlockINTEL( CallInst *CI, StringRef DemangledName) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); spv::Op OpCode = DemangledName.rfind("read") != StringRef::npos ? spv::OpSubgroupImageMediaBlockReadINTEL : spv::OpSubgroupImageMediaBlockWriteINTEL; mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { // Moving the last argument to the beginning. std::rotate(Args.begin(), Args.end() - 1, Args.end()); return getSPIRVFuncName(OpCode, CI->getType()); }, &Attrs); } static const char *getSubgroupAVCIntelOpKind(StringRef Name) { return StringSwitch(Name.data()) .StartsWith(kOCLSubgroupsAVCIntel::IMEPrefix, "ime") .StartsWith(kOCLSubgroupsAVCIntel::REFPrefix, "ref") .StartsWith(kOCLSubgroupsAVCIntel::SICPrefix, "sic"); } static const char *getSubgroupAVCIntelTyKind(StringRef MangledName) { // We're looking for the type name of the last parameter, which will be at the // very end of the mangled name. Since we only care about the ending of the // name, we don't need to be any more clever than this. return MangledName.endswith("_payload_t") ? "payload" : "result"; } static Type *getSubgroupAVCIntelMCEType(Module *M, std::string &TName) { auto Ty = StructType::getTypeByName(M->getContext(), TName); if (Ty) return Ty; return StructType::create(M->getContext(), TName); } static Op getSubgroupAVCIntelMCEOpCodeForWrapper(StringRef DemangledName) { if (DemangledName.size() <= strlen(kOCLSubgroupsAVCIntel::MCEPrefix)) return OpNop; // this is not a VME built-in std::string MCEName{DemangledName}; MCEName.replace(0, strlen(kOCLSubgroupsAVCIntel::MCEPrefix), kOCLSubgroupsAVCIntel::MCEPrefix); Op MCEOC = OpNop; OCLSPIRVSubgroupAVCIntelBuiltinMap::find(MCEName, &MCEOC); return MCEOC; } // Handles Subgroup AVC Intel extension generic built-ins. void OCLToSPIRVBase::visitSubgroupAVCBuiltinCall(CallInst *CI, StringRef DemangledName) { Op OC = OpNop; std::string FName{DemangledName}; std::string Prefix = kOCLSubgroupsAVCIntel::Prefix; // Update names for built-ins mapped on two or more SPIRV instructions if (FName.find(Prefix + "ime_get_streamout_major_shape_") == 0) { // _single_reference functions have 2 arguments, _dual_reference have 3 // arguments. FName += (CI->arg_size() == 2) ? "_single_reference" : "_dual_reference"; } else if (FName.find(Prefix + "sic_configure_ipe") == 0) { FName += (CI->arg_size() == 8) ? "_luma" : "_luma_chroma"; } OCLSPIRVSubgroupAVCIntelBuiltinMap::find(FName, &OC); if (OC == OpNop) { if (Op MCEOC = getSubgroupAVCIntelMCEOpCodeForWrapper(DemangledName)) // The called function is a VME wrapper built-in return visitSubgroupAVCWrapperBuiltinCall(CI, MCEOC, DemangledName); else // The called function isn't a VME built-in return; } AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { return getSPIRVFuncName(OC); }, &Attrs); } // Handles Subgroup AVC Intel extension wrapper built-ins. // 'IME', 'REF' and 'SIC' sets contain wrapper built-ins which don't have // corresponded instructions in SPIRV and should be translated to a // conterpart from 'MCE' with conversion for an argument and result (if needed). void OCLToSPIRVBase::visitSubgroupAVCWrapperBuiltinCall( CallInst *CI, Op WrappedOC, StringRef DemangledName) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); std::string Prefix = kOCLSubgroupsAVCIntel::Prefix; // Find 'to_mce' conversion function. // The operand required conversion is always the last one. const char *OpKind = getSubgroupAVCIntelOpKind(DemangledName); const char *TyKind = getSubgroupAVCIntelTyKind(CI->getCalledFunction()->getName()); std::string MCETName = std::string(kOCLSubgroupsAVCIntel::TypePrefix) + "mce_" + TyKind + "_t"; auto *MCESTy = getSubgroupAVCIntelMCEType(M, MCETName); auto *MCETy = PointerType::get(MCESTy, SPIRAS_Private); std::string ToMCEFName = Prefix + OpKind + "_convert_to_mce_" + TyKind; Op ToMCEOC = OpNop; OCLSPIRVSubgroupAVCIntelBuiltinMap::find(ToMCEFName, &ToMCEOC); assert(ToMCEOC != OpNop && "Invalid Subgroup AVC Intel built-in call"); SmallVector ParamTys; getParameterTypes(CI, ParamTys); if (std::strcmp(TyKind, "payload") == 0) { // Wrapper built-ins which take the 'payload_t' argument return it as // the result: two conversion calls required. std::string FromMCEFName = Prefix + "mce_convert_to_" + OpKind + "_" + TyKind; Op FromMCEOC = OpNop; OCLSPIRVSubgroupAVCIntelBuiltinMap::find(FromMCEFName, &FromMCEOC); assert(FromMCEOC != OpNop && "Invalid Subgroup AVC Intel built-in call"); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args, Type *&Ret) { Ret = MCETy; // Create conversion function call for the last operand Args[Args.size() - 1] = addCallInstSPIRV( M, getSPIRVFuncName(ToMCEOC), MCETy, Args[Args.size() - 1], nullptr, {ParamTys[Args.size() - 1]}, CI, ""); return getSPIRVFuncName(WrappedOC); }, [=](CallInst *NewCI) -> Instruction * { // Create conversion function call for the return result return addCallInstSPIRV(M, getSPIRVFuncName(FromMCEOC), CI->getType(), NewCI, nullptr, {MCESTy}, CI, ""); }, &Attrs); } else { // Wrapper built-ins which take the 'result_t' argument requires only one // conversion for the argument mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { // Create conversion function call for the last // operand Args[Args.size() - 1] = addCallInstSPIRV( M, getSPIRVFuncName(ToMCEOC), MCETy, Args[Args.size() - 1], nullptr, {ParamTys[Args.size() - 1]}, CI, ""); return getSPIRVFuncName(WrappedOC); }, &Attrs); } } // Handles Subgroup AVC Intel extension built-ins which take sampler as // an argument (their SPIR-V counterparts take OpTypeVmeImageIntel instead) void OCLToSPIRVBase::visitSubgroupAVCBuiltinCallWithSampler( CallInst *CI, StringRef DemangledName) { std::string FName{DemangledName}; std::string Prefix = kOCLSubgroupsAVCIntel::Prefix; // Update names for built-ins mapped on two or more SPIRV instructions if (FName.find(Prefix + "ref_evaluate_with_multi_reference") == 0 || FName.find(Prefix + "sic_evaluate_with_multi_reference") == 0) { FName += (CI->arg_size() == 5) ? "_interlaced" : ""; } Op OC = OpNop; OCLSPIRVSubgroupAVCIntelBuiltinMap::find(FName, &OC); if (OC == OpNop) return; // this is not a VME built-in AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { SmallVector ParamTys; getParameterTypes(CI, ParamTys); auto *TyIt = std::find_if(ParamTys.begin(), ParamTys.end(), isSamplerStructTy); assert(TyIt != ParamTys.end() && "Invalid Subgroup AVC Intel built-in call"); auto SamplerIt = Args.begin() + (TyIt - ParamTys.begin()); auto *SamplerVal = *SamplerIt; auto *SamplerTy = *TyIt; Args.erase(SamplerIt); ParamTys.erase(TyIt); for (unsigned I = 0, E = Args.size(); I < E; ++I) { if (!isOCLImageStructType(ParamTys[I])) continue; auto *ImageTy = OCLTypeToSPIRVPtr ->getAdaptedArgumentType(CI->getCalledFunction(), I) .second; if (!ImageTy) ImageTy = ParamTys[I]; ImageTy = adaptSPIRVImageType(M, ImageTy); auto *SampledImgTy = getSPIRVTypeByChangeBaseTypeName( M, ImageTy, kSPIRVTypeName::Image, kSPIRVTypeName::VmeImageINTEL); Value *SampledImgArgs[] = {Args[I], SamplerVal}; Args[I] = addCallInstSPIRV(M, getSPIRVFuncName(OpVmeImageINTEL), SampledImgTy, SampledImgArgs, nullptr, {ParamTys[I], SamplerTy}, CI, kSPIRVName::TempSampledImage); } return getSPIRVFuncName(OC); }, &Attrs); } void OCLToSPIRVBase::visitCallSplitBarrierINTEL(CallInst *CI, StringRef DemangledName) { auto Lit = getBarrierLiterals(CI); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Op OpCode = StringSwitch(DemangledName) .Case("intel_work_group_barrier_arrive", OpControlBarrierArriveINTEL) .Case("intel_work_group_barrier_wait", OpControlBarrierWaitINTEL) .Default(OpNop); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(3); // Execution scope Args[0] = addInt32(map(std::get<2>(Lit))); // Memory scope Args[1] = addInt32(map(std::get<1>(Lit))); // Memory semantics // OpControlBarrierArriveINTEL -> Release, // OpControlBarrierWaitINTEL -> Acquire unsigned MemFenceFlag = std::get<0>(Lit); OCLMemOrderKind MemOrder = OpCode == OpControlBarrierArriveINTEL ? OCLMO_release : OCLMO_acquire; Args[2] = addInt32(mapOCLMemSemanticToSPIRV(MemFenceFlag, MemOrder)); return getSPIRVFuncName(OpCode); }, &Attrs); } void OCLToSPIRVBase::visitCallLdexp(CallInst *CI, StringRef MangledName, StringRef DemangledName) { auto Args = getArguments(CI); if (Args.size() == 2) { Type *Type0 = Args[0]->getType(); Type *Type1 = Args[1]->getType(); // For OpenCL built-in math functions 'halfn ldexp(halfn x, int k)', // 'floatn ldexp(floatn x, int k)' and 'doublen ldexp (doublen x, int k)', // convert scalar arg to vector to keep consistency with SPIRV spec. // Regarding to SPIRV OpenCL Extended Instruction set, k operand must have // the same component count as Result Type and x operands if (auto *FixedVecType0 = dyn_cast(Type0)) { auto ScalarTypeID = Type0->getScalarType()->getTypeID(); if ((ScalarTypeID == llvm::Type::FloatTyID || ScalarTypeID == llvm::Type::DoubleTyID || ScalarTypeID == llvm::Type::HalfTyID) && Type1->isIntegerTy()) { IRBuilder<> IRB(CI); unsigned Width = FixedVecType0->getNumElements(); CI->setOperand(1, IRB.CreateVectorSplat(Width, CI->getArgOperand(1))); } } } visitCallBuiltinSimple(CI, MangledName, DemangledName); } void OCLToSPIRVBase::visitCallConvertBFloat16AsUshort(CallInst *CI, StringRef DemangledName) { Type *RetTy = CI->getType(); Type *ArgTy = CI->getOperand(0)->getType(); if (DemangledName == kOCLBuiltinName::ConvertBFloat16AsUShort) { if (!RetTy->isIntegerTy(16U) || !ArgTy->isFloatTy()) report_fatal_error( "OpConvertBFloat16AsUShort must be of i16 and take float"); } else { FixedVectorType *RetTyVec = cast(RetTy); FixedVectorType *ArgTyVec = cast(ArgTy); if (!RetTyVec || !RetTyVec->getElementType()->isIntegerTy(16U) || !ArgTyVec || !ArgTyVec->getElementType()->isFloatTy()) report_fatal_error("OpConvertBFloat16NAsUShortN must be of and " "take "); unsigned RetTyVecSize = RetTyVec->getNumElements(); unsigned ArgTyVecSize = ArgTyVec->getNumElements(); if (DemangledName == kOCLBuiltinName::ConvertBFloat162AsUShort2) { if (RetTyVecSize != 2 || ArgTyVecSize != 2) report_fatal_error("ConvertBFloat162AsUShort2 must be of <2 x i16> and " "take <2 x float>"); } else if (DemangledName == kOCLBuiltinName::ConvertBFloat163AsUShort3) { if (RetTyVecSize != 3 || ArgTyVecSize != 3) report_fatal_error("ConvertBFloat163AsUShort3 must be of <3 x i16> and " "take <3 x float>"); } else if (DemangledName == kOCLBuiltinName::ConvertBFloat164AsUShort4) { if (RetTyVecSize != 4 || ArgTyVecSize != 4) report_fatal_error("ConvertBFloat164AsUShort4 must be of <4 x i16> and " "take <4 x float>"); } else if (DemangledName == kOCLBuiltinName::ConvertBFloat168AsUShort8) { if (RetTyVecSize != 8 || ArgTyVecSize != 8) report_fatal_error("ConvertBFloat168AsUShort8 must be of <8 x i16> and " "take <8 x float>"); } else if (DemangledName == kOCLBuiltinName::ConvertBFloat1616AsUShort16) { if (RetTyVecSize != 16 || ArgTyVecSize != 16) report_fatal_error("ConvertBFloat1616AsUShort16 must be of <16 x i16> " "and take <16 x float>"); } } AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { return getSPIRVFuncName(internal::OpConvertFToBF16INTEL); }, &Attrs); } void OCLToSPIRVBase::visitCallConvertAsBFloat16Float(CallInst *CI, StringRef DemangledName) { Type *RetTy = CI->getType(); Type *ArgTy = CI->getOperand(0)->getType(); if (DemangledName == kOCLBuiltinName::ConvertAsBFloat16Float) { if (!RetTy->isFloatTy() || !ArgTy->isIntegerTy(16U)) report_fatal_error( "OpConvertAsBFloat16Float must be of float and take i16"); } else { FixedVectorType *RetTyVec = cast(RetTy); FixedVectorType *ArgTyVec = cast(ArgTy); if (!RetTyVec || !RetTyVec->getElementType()->isFloatTy() || !ArgTyVec || !ArgTyVec->getElementType()->isIntegerTy(16U)) report_fatal_error("OpConvertAsBFloat16NFloatN must be of " "and take "); unsigned RetTyVecSize = RetTyVec->getNumElements(); unsigned ArgTyVecSize = ArgTyVec->getNumElements(); if (DemangledName == kOCLBuiltinName::ConvertAsBFloat162Float2) { if (RetTyVecSize != 2 || ArgTyVecSize != 2) report_fatal_error("ConvertAsBFloat162Float2 must be of <2 x float> " "and take <2 x i16>"); } else if (DemangledName == kOCLBuiltinName::ConvertAsBFloat163Float3) { if (RetTyVecSize != 3 || ArgTyVecSize != 3) report_fatal_error("ConvertAsBFloat163Float3 must be of <3 x float> " "and take <3 x i16>"); } else if (DemangledName == kOCLBuiltinName::ConvertAsBFloat164Float4) { if (RetTyVecSize != 4 || ArgTyVecSize != 4) report_fatal_error("ConvertAsBFloat164Float4 must be of <4 x float> " "and take <4 x i16>"); } else if (DemangledName == kOCLBuiltinName::ConvertAsBFloat168Float8) { if (RetTyVecSize != 8 || ArgTyVecSize != 8) report_fatal_error("ConvertAsBFloat168Float8 must be of <8 x float> " "and take <8 x i16>"); } else if (DemangledName == kOCLBuiltinName::ConvertAsBFloat1616Float16) { if (RetTyVecSize != 16 || ArgTyVecSize != 16) report_fatal_error("ConvertAsBFloat1616Float16 must be of <16 x float> " "and take <16 x i16>"); } } AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstSPIRV( M, CI, [=](CallInst *, std::vector &Args) { return getSPIRVFuncName(internal::OpConvertBF16ToFINTEL); }, &Attrs); } } // namespace SPIRV INITIALIZE_PASS_BEGIN(OCLToSPIRVLegacy, "ocl-to-spv", "Transform OCL 2.0 to SPIR-V", false, false) INITIALIZE_PASS_DEPENDENCY(OCLTypeToSPIRVLegacy) INITIALIZE_PASS_END(OCLToSPIRVLegacy, "ocl-to-spv", "Transform OCL 2.0 to SPIR-V", false, false) ModulePass *llvm::createOCLToSPIRVLegacy() { return new OCLToSPIRVLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLToSPIRV.h000066400000000000000000000317731506646577400211140ustar00rootroot00000000000000//=- OCLToSPIRV.h - OpenCL to SPIR-V builtin preprocessing pass -*- C++ -*-=// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements preprocessing of OpenCL C built-in functions into SPIR-V // friendly IR form for further translation into SPIR-V // //===----------------------------------------------------------------------===// #ifndef SPIRV_OCLTOSPIRV_H #define SPIRV_OCLTOSPIRV_H #include "OCLUtil.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class OCLTypeToSPIRVBase; class OCLToSPIRVBase : public InstVisitor { public: OCLToSPIRVBase() : M(nullptr), Ctx(nullptr), CLVer(0), OCLTypeToSPIRVPtr(nullptr) {} virtual ~OCLToSPIRVBase() {} bool runOCLToSPIRV(Module &M); virtual void visitCallInst(CallInst &CI); /// Transform barrier/work_group_barrier/sub_group_barrier /// to __spirv_ControlBarrier. /// barrier(flag) => /// __spirv_ControlBarrier(workgroup, workgroup, map(flag)) /// work_group_barrier(scope, flag) => /// __spirv_ControlBarrier(workgroup, map(scope), map(flag)) /// sub_group_barrier(scope, flag) => /// __spirv_ControlBarrier(subgroup, map(scope), map(flag)) void visitCallBarrier(CallInst *CI); /// Erase useless convert functions. /// \return true if the call instruction is erased. bool eraseUselessConvert(CallInst *Call, StringRef MangledName, StringRef DeMangledName); /// Transform convert_ to /// __spirv_{CastOpName}_R{TargeTyName}{_sat}{_rt[p|n|z|e]} void visitCallConvert(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform async_work_group{_strided}_copy. /// async_work_group_copy(dst, src, n, event) /// => async_work_group_strided_copy(dst, src, n, 1, event) /// async_work_group_strided_copy(dst, src, n, stride, event) /// => __spirv_AsyncGroupCopy(ScopeWorkGroup, dst, src, n, stride, event) void visitCallAsyncWorkGroupCopy(CallInst *CI, StringRef DemangledName); /// Transform OCL builtin function to SPIR-V builtin function. void transBuiltin(CallInst *CI, OCLBuiltinTransInfo &Info); /// Transform atomic_work_item_fence/mem_fence to __spirv_MemoryBarrier. /// func(flag, order, scope) => /// __spirv_MemoryBarrier(map(scope), map(flag)|map(order)) void transMemoryBarrier(CallInst *CI, AtomicWorkItemFenceLiterals); /// Transform all to __spirv_Op(All|Any). Note that the types mismatch so // some extra code is emitted to convert between the two. void visitCallAllAny(spv::Op OC, CallInst *CI); /// Transform atomic_* to __spirv_Atomic*. /// atomic_x(ptr_arg, args, order, scope) => /// __spirv_AtomicY(ptr_arg, map(order), map(scope), args) void transAtomicBuiltin(CallInst *CI, OCLBuiltinTransInfo &Info); /// Transform atomic_work_item_fence to __spirv_MemoryBarrier. /// atomic_work_item_fence(flag, order, scope) => /// __spirv_MemoryBarrier(map(scope), map(flag)|map(order)) void visitCallAtomicWorkItemFence(CallInst *CI); /// Transform atomic_compare_exchange call. /// In atomic_compare_exchange, the expected value parameter is a pointer. /// However in SPIR-V it is a value. The transformation adds a load /// instruction, result of which is passed to atomic_compare_exchange as /// argument. /// The transformation adds a store instruction after the call, to update the /// value in expected with the value pointed to by object. Though, it is not /// necessary in case they are equal, this approach makes result code simpler. /// Also ICmp instruction is added, because the call must return result of /// comparison. /// \returns the call instruction of atomic_compare_exchange_strong. CallInst *visitCallAtomicCmpXchg(CallInst *CI); /// Transform atomic_init. /// atomic_init(p, x) => store p, x void visitCallAtomicInit(CallInst *CI); /// Transform legacy OCL 1.x atomic builtins to SPIR-V builtins for extensions /// cl_khr_int64_base_atomics /// cl_khr_int64_extended_atomics /// Do nothing if the called function is not a legacy atomic builtin. void visitCallAtomicLegacy(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform OCL 2.0 C++11 atomic builtins to SPIR-V builtins. /// Do nothing if the called function is not a C++11 atomic builtin. void visitCallAtomicCpp11(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform OCL builtin function to SPIR-V builtin function. /// Assuming there is a simple name mapping without argument changes. /// Should be called at last. void visitCallBuiltinSimple(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform get_image_{width|height|depth|dim}. /// get_image_xxx(...) => /// dimension = __spirv_ImageQuerySizeLod_R{ReturnType}(...); /// return dimension.{x|y|z}; void visitCallGetImageSize(CallInst *CI, StringRef DemangledName); /// Transform {work|sub}_group_x => /// __spirv_{OpName} /// /// Special handling of work_group_broadcast. /// work_group_broadcast(a, x, y, z) /// => /// __spirv_GroupBroadcast(a, vec3(x, y, z)) void visitCallGroupBuiltin(CallInst *CI, StringRef DemangledName); /// Transform mem_fence to __spirv_MemoryBarrier. /// mem_fence(flag) => __spirv_MemoryBarrier(Workgroup, map(flag)) void visitCallMemFence(CallInst *CI, StringRef DemangledName); void visitCallNDRange(CallInst *CI, StringRef DemangledName); /// Transform read_image with sampler arguments. /// read_image(image, sampler, ...) => /// sampled_image = __spirv_SampledImage(image, sampler); /// return __spirv_ImageSampleExplicitLod_R{ReturnType}(sampled_image, ...); void visitCallReadImageWithSampler(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform read_image with msaa image arguments. /// Sample argument must be acoded as Image Operand. void visitCallReadImageMSAA(CallInst *CI, StringRef MangledName); /// Transform {read|write}_image without sampler arguments. void visitCallReadWriteImage(CallInst *CI, StringRef DemangledName); /// Transform to_{global|local|private}. /// /// T* a = ...; /// addr T* b = to_addr(a); /// => /// i8* x = cast(a); /// addr i8* y = __spirv_GenericCastToPtr_ToAddr(x); /// addr T* b = cast(y); void visitCallToAddr(CallInst *CI, StringRef DemangledName); /// Transform return type of relatinal built-in functions like isnan, isfinite /// to boolean values. void visitCallRelational(CallInst *CI, StringRef DemangledName); /// Transform vector load/store functions to SPIR-V extended builtin /// functions /// {vload|vstore{a}}{_half}{n}{_rte|_rtz|_rtp|_rtn} => /// __spirv_ocl_{ExtendedInstructionOpCodeName}__R{ReturnType} void visitCallVecLoadStore(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transforms get_mem_fence built-in to SPIR-V function and aligns result /// values with SPIR 1.2. get_mem_fence(ptr) => __spirv_GenericPtrMemSemantics /// GenericPtrMemSemantics valid values are 0x100, 0x200 and 0x300, where is /// SPIR 1.2 defines them as 0x1, 0x2 and 0x3, so this function adjusts /// GenericPtrMemSemantics results to SPIR 1.2 values. void visitCallGetFence(CallInst *CI, StringRef DemangledName); /// Transforms OpDot instructions with a scalar type to a fmul instruction void visitCallDot(CallInst *CI); /// Transforms OpDot instructions with a vector or scalar (packed vector) type /// to dot or dot_acc_sat instructions void visitCallDot(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform clock_read_* calls to OpReadClockKHR instructions. void visitCallClockRead(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Fixes for built-in functions with vector+scalar arguments that are /// translated to the SPIR-V instructions where all arguments must have the /// same type. void visitCallScalToVec(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// Transform get_image_channel_{order|data_type} built-in functions to /// __spirv_ocl_{ImageQueryOrder|ImageQueryFormat} void visitCallGetImageChannel(CallInst *CI, StringRef DemangledName, unsigned int Offset); /// Transform enqueue_kernel and kernel query built-in functions to /// spirv-friendly format filling arguments, required for device-side enqueue /// instructions, but missed in the original call void visitCallEnqueueKernel(CallInst *CI, StringRef DemangledName); void visitCallKernelQuery(CallInst *CI, StringRef DemangledName); /// For cl_intel_subgroups block read built-ins: void visitSubgroupBlockReadINTEL(CallInst *CI); /// For cl_intel_subgroups block write built-ins: void visitSubgroupBlockWriteINTEL(CallInst *CI); /// For cl_intel_media_block_io built-ins: void visitSubgroupImageMediaBlockINTEL(CallInst *CI, StringRef DemangledName); // For cl_intel_device_side_avc_motion_estimation built-ins void visitSubgroupAVCBuiltinCall(CallInst *CI, StringRef DemangledName); void visitSubgroupAVCWrapperBuiltinCall(CallInst *CI, Op WrappedOC, StringRef DemangledName); void visitSubgroupAVCBuiltinCallWithSampler(CallInst *CI, StringRef DemangledName); /// For cl_intel_split_work_group_barrier built-ins: void visitCallSplitBarrierINTEL(CallInst *CI, StringRef DemangledName); void visitCallLdexp(CallInst *CI, StringRef MangledName, StringRef DemangledName); /// For cl_intel_convert_bfloat16_as_ushort void visitCallConvertBFloat16AsUshort(CallInst *CI, StringRef DemangledName); /// For cl_intel_convert_as_bfloat16_float void visitCallConvertAsBFloat16Float(CallInst *CI, StringRef DemangledName); void setOCLTypeToSPIRV(OCLTypeToSPIRVBase *OCLTypeToSPIRV) { OCLTypeToSPIRVPtr = OCLTypeToSPIRV; } OCLTypeToSPIRVBase *getOCLTypeToSPIRV() { return OCLTypeToSPIRVPtr; } private: Module *M; LLVMContext *Ctx; unsigned CLVer; /// OpenCL version as major*10+minor std::set ValuesToDelete; OCLTypeToSPIRVBase *OCLTypeToSPIRVPtr; ConstantInt *addInt32(int I) { return getInt32(M, I); } ConstantInt *addSizet(uint64_t I) { return getSizet(M, I); } /// Get vector width from OpenCL vload* function name. SPIRVWord getVecLoadWidth(const std::string &DemangledName); /// Transform OpenCL vload/vstore function name. void transVecLoadStoreName(std::string &DemangledName, const std::string &Stem, bool AlwaysN); }; class OCLToSPIRVLegacy : public OCLToSPIRVBase, public llvm::ModulePass { public: OCLToSPIRVLegacy() : ModulePass(ID) { initializeOCLToSPIRVLegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override; void getAnalysisUsage(AnalysisUsage &AU) const override; static char ID; }; class OCLToSPIRVPass : public OCLToSPIRVBase, public llvm::PassInfoMixin { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; } // namespace SPIRV #endif // SPIRV_OCLTOSPIRV_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLTypeToSPIRV.cpp000066400000000000000000000261741506646577400223100ustar00rootroot00000000000000//===- OCLTypeToSPIRV.cpp - Adapt types from OCL for SPIRV ------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements adaptation of OCL types for SPIR-V. // // It first maps kernel arguments of OCL opaque types to SPIR-V type, then // propagates the mapping to the uses of the kernel arguments. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "cltytospv" #include "OCLTypeToSPIRV.h" #include "OCLUtil.h" #include "SPIRVInternal.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include #include using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { char OCLTypeToSPIRVLegacy::ID = 0; OCLTypeToSPIRVLegacy::OCLTypeToSPIRVLegacy() : ModulePass(ID) { initializeOCLTypeToSPIRVLegacyPass(*PassRegistry::getPassRegistry()); } void OCLTypeToSPIRVLegacy::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); } bool OCLTypeToSPIRVLegacy::runOnModule(Module &M) { return runOCLTypeToSPIRV(M); } OCLTypeToSPIRVBase &OCLTypeToSPIRVPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { runOCLTypeToSPIRV(M); return *this; } OCLTypeToSPIRVBase::OCLTypeToSPIRVBase() : M(nullptr), Ctx(nullptr) {} bool OCLTypeToSPIRVBase::runOCLTypeToSPIRV(Module &Module) { LLVM_DEBUG(dbgs() << "Enter OCLTypeToSPIRV:\n"); M = &Module; Ctx = &M->getContext(); AdaptedTy.clear(); WorkSet.clear(); auto Src = getSPIRVSource(&Module); if (std::get<0>(Src) != spv::SourceLanguageOpenCL_C) return false; for (auto &F : Module.functions()) adaptArgumentsByMetadata(&F); for (auto &F : Module.functions()) adaptFunctionArguments(&F); adaptArgumentsBySamplerUse(Module); while (!WorkSet.empty()) { Function *F = *WorkSet.begin(); WorkSet.erase(WorkSet.begin()); adaptFunction(F); } return false; } void OCLTypeToSPIRVBase::addAdaptedType(Value *V, Type *Ty, unsigned AddrSpace) { LLVM_DEBUG(dbgs() << "[add adapted type] "; V->printAsOperand(dbgs(), true, M); dbgs() << " => " << *Ty << '\n'); AdaptedTy[V] = {Ty, AddrSpace}; } void OCLTypeToSPIRVBase::addWork(Function *F) { LLVM_DEBUG(dbgs() << "[add work] "; F->printAsOperand(dbgs(), true, M); dbgs() << '\n'); WorkSet.insert(F); } /// Create a new function type if \param F has arguments in AdaptedTy, and /// propagates the adapted arguments to functions called by \param F. void OCLTypeToSPIRVBase::adaptFunction(Function *F) { LLVM_DEBUG(dbgs() << "\n[work on function] "; F->printAsOperand(dbgs(), true, M); dbgs() << '\n'); assert(AdaptedTy.count(F) == 0); std::vector ArgTys; bool Changed = false; for (auto &I : F->args()) { auto Loc = AdaptedTy.find(&I); auto Found = (Loc != AdaptedTy.end()); Changed |= Found; ArgTys.push_back(Found ? Loc->second.first : I.getType()); if (Found) { auto *Ty = Loc->second.first; unsigned AddrSpace = Loc->second.second; for (auto &U : I.uses()) { if (auto *CI = dyn_cast(U.getUser())) { auto ArgIndex = CI->getArgOperandNo(&U); auto CF = CI->getCalledFunction(); if (AdaptedTy.count(CF) == 0) { addAdaptedType(CF->getArg(ArgIndex), Ty, AddrSpace); addWork(CF); } } } } } if (!Changed) return; auto FT = F->getFunctionType(); FT = FunctionType::get(FT->getReturnType(), ArgTys, FT->isVarArg()); addAdaptedType(F, FT, 0); } // Handle functions with sampler arguments that don't get called by // a kernel function. void OCLTypeToSPIRVBase::adaptArgumentsBySamplerUse(Module &M) { SmallPtrSet Processed; std::function TraceArg = [&](Function *F, unsigned Idx) { // If we have cycles in the call graph in the future, bail out // if we've already processed this function. if (Processed.insert(F).second == false) return; for (auto U : F->users()) { auto *CI = dyn_cast(U); if (!CI) continue; auto SamplerArg = CI->getArgOperand(Idx); if (!isa(SamplerArg) || AdaptedTy.count(SamplerArg) != 0) // Already traced this, move on. continue; addAdaptedType(SamplerArg, getSamplerStructType(&M), SPIRAS_Constant); auto Caller = cast(SamplerArg)->getParent(); addWork(Caller); TraceArg(Caller, cast(SamplerArg)->getArgNo()); } }; for (auto &F : M) { if (!F.empty()) // not decl continue; auto MangledName = F.getName(); StringRef DemangledName; if (!oclIsBuiltin(MangledName, DemangledName, false)) continue; // Note: kSPIRVName::ConvertHandleToSampledImageINTEL contains // kSPIRVName::SampledImage as a substring, but we still want to continue in // this case. if (DemangledName.find(kSPIRVName::SampledImage) == std::string::npos || DemangledName.find(kSPIRVName::ConvertHandleToSampledImageINTEL) != std::string::npos) continue; TraceArg(&F, 1); } } void OCLTypeToSPIRVBase::adaptFunctionArguments(Function *F) { auto TypeMD = F->getMetadata(SPIR_MD_KERNEL_ARG_BASE_TYPE); if (TypeMD) return; bool Changed = false; auto Arg = F->arg_begin(); SmallVector ParamTys; getParameterTypes(F, ParamTys); // If we couldn't get any information from demangling, there is nothing that // can be done. if (ParamTys.empty()) return; for (unsigned I = 0; I < F->arg_size(); ++I, ++Arg) { StructType *NewTy = ParamTys[I]; if (NewTy && NewTy->isOpaque()) { auto STName = NewTy->getStructName(); if (!hasAccessQualifiedName(STName)) continue; if (STName.startswith(kSPR2TypeName::ImagePrefix)) { auto Ty = STName.str(); auto AccStr = getAccessQualifierFullName(Ty); addAdaptedType( &*Arg, getOrCreateOpaqueStructType(M, mapOCLTypeNameToSPIRV(Ty, AccStr)), SPIRAS_Global); Changed = true; } } } if (Changed) addWork(F); } /// Go through all kernel functions, get access qualifier for image and pipe /// types and use them to map the function arguments to the SPIR-V type. /// ToDo: Map other OpenCL opaque types to SPIR-V types. void OCLTypeToSPIRVBase::adaptArgumentsByMetadata(Function *F) { auto TypeMD = F->getMetadata(SPIR_MD_KERNEL_ARG_BASE_TYPE); if (!TypeMD) return; bool Changed = false; auto Arg = F->arg_begin(); for (unsigned I = 0, E = TypeMD->getNumOperands(); I != E; ++I, ++Arg) { auto OCLTyStr = getMDOperandAsString(TypeMD, I); if (OCLTyStr == OCL_TYPE_NAME_SAMPLER_T) { addAdaptedType(&(*Arg), getSamplerStructType(M), SPIRAS_Constant); Changed = true; } else if (OCLTyStr.startswith("image") && OCLTyStr.endswith("_t")) { auto Ty = (Twine("opencl.") + OCLTyStr).str(); if (StructType::getTypeByName(F->getContext(), Ty)) { auto AccMD = F->getMetadata(SPIR_MD_KERNEL_ARG_ACCESS_QUAL); assert(AccMD && "Invalid access qualifier metadata"); auto AccStr = getMDOperandAsString(AccMD, I); addAdaptedType( &(*Arg), getOrCreateOpaqueStructType(M, mapOCLTypeNameToSPIRV(Ty, AccStr)), SPIRAS_Global); Changed = true; } } } if (Changed) addWork(F); } // OCL sampler, image and pipe type need to be regularized before converting // to SPIRV types. // // OCL sampler type is represented as i32 in LLVM, however in SPIRV it is // represented as OpTypeSampler. Also LLVM uses the same pipe type to // represent pipe types with different underlying data types, however // in SPIRV they are different types. OCL image and pipe types do not // encode access qualifier, which is part of SPIRV types for image and pipe. // // The function types in LLVM need to be regularized before translating // to SPIRV function types: // // sampler type as i32 -> opencl.sampler_t opaque type // opencl.pipe_t opaque type with underlying opencl type x and access // qualifier y -> opencl.pipe_t.x.y opaque type // opencl.image_x opaque type with access qualifier y -> // opencl.image_x.y opaque type // // The converter relies on kernel_arg_base_type to identify the sampler // type, the underlying data type of pipe type, and access qualifier for // image and pipe types. The FE is responsible to generate the correct // kernel_arg_base_type metadata. // // Alternatively,the FE may choose to use opencl.sampler_t to represent // sampler type, use opencl.pipe_t.x.y to represent pipe type with underlying // opencl data type x and access qualifier y, and use opencl.image_x.y to // represent image_x type with access qualifier y. // std::pair OCLTypeToSPIRVBase::getAdaptedArgumentType(Function *F, unsigned ArgNo) { Value *Arg = F->getArg(ArgNo); auto Loc = AdaptedTy.find(Arg); if (Loc == AdaptedTy.end()) return {nullptr, nullptr}; Type *PointeeTy = Loc->second.first; Type *PointerTy = PointerType::get(PointeeTy, Loc->second.second); return {PointerTy, PointeeTy}; } } // namespace SPIRV AnalysisKey OCLTypeToSPIRVPass::Key; INITIALIZE_PASS(OCLTypeToSPIRVLegacy, "cltytospv", "Adapt OCL types for SPIR-V", false, true) ModulePass *llvm::createOCLTypeToSPIRVLegacy() { return new OCLTypeToSPIRVLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLTypeToSPIRV.h000066400000000000000000000101411506646577400217400ustar00rootroot00000000000000//===- OCLTypeToSPIRV.h - Adapt types from OCL for SPIRV --------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements adaptation of OCL types for SPIRV. It does not modify // the module. Instead, it returns adapted function type based on kernel // argument metadata. Later LLVM/SPIRV translator will translate the adapted // type instead of the original type. // //===----------------------------------------------------------------------===// #ifndef SPIRV_OCLTYPETOSPIRV_H #define SPIRV_OCLTYPETOSPIRV_H #include "LLVMSPIRVLib.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include #include namespace SPIRV { class OCLTypeToSPIRVBase { public: OCLTypeToSPIRVBase(); bool runOCLTypeToSPIRV(llvm::Module &M); /// Returns the adapted type of the corresponding argument for a function. /// The first value of the returned pair is the LLVM type of the argument. /// The second value of the returned pair is the pointer element type of the /// argument, if the type is a pointer. std::pair getAdaptedArgumentType(llvm::Function *F, unsigned ArgNo); private: llvm::Module *M; llvm::LLVMContext *Ctx; // Map of argument/Function -> {pointee type, address space} std::map> AdaptedTy; std::set WorkSet; // Functions to be adapted void adaptFunctionArguments(llvm::Function *F); void adaptArgumentsByMetadata(llvm::Function *F); void adaptArgumentsBySamplerUse(llvm::Module &M); void adaptFunction(llvm::Function *F); void addAdaptedType(llvm::Value *V, llvm::Type *PointeeTy, unsigned AS); void addWork(llvm::Function *F); }; class OCLTypeToSPIRVLegacy : public OCLTypeToSPIRVBase, public llvm::ModulePass { public: OCLTypeToSPIRVLegacy(); void getAnalysisUsage(llvm::AnalysisUsage &AU) const override; bool runOnModule(llvm::Module &M) override; static char ID; }; class OCLTypeToSPIRVPass : public OCLTypeToSPIRVBase, public llvm::AnalysisInfoMixin { public: using Result = OCLTypeToSPIRVBase; static llvm::AnalysisKey Key; OCLTypeToSPIRVBase &run(llvm::Module &F, llvm::ModuleAnalysisManager &MAM); }; } // namespace SPIRV #endif // SPIRV_OCLTYPETOSPIRV_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLUtil.cpp000066400000000000000000002020501506646577400211420ustar00rootroot00000000000000//===- OCLUtil.cpp - OCL Utilities ----------------------------------------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements OCL utility functions. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "oclutil" #include "OCLUtil.h" #include "SPIRVEntry.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVInternal.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" using namespace llvm; using namespace SPIRV; namespace OCLUtil { #ifndef SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #define SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE SPIRAS_Private #endif #ifndef SPIRV_QUEUE_T_ADDR_SPACE #define SPIRV_QUEUE_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #endif #ifndef SPIRV_EVENT_T_ADDR_SPACE #define SPIRV_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #endif #ifndef SPIRV_AVC_INTEL_T_ADDR_SPACE #define SPIRV_AVC_INTEL_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #endif #ifndef SPIRV_CLK_EVENT_T_ADDR_SPACE #define SPIRV_CLK_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #endif #ifndef SPIRV_SAMPLER_T_ADDR_SPACE #define SPIRV_SAMPLER_T_ADDR_SPACE SPIRAS_Constant #endif #ifndef SPIRV_RESERVE_ID_T_ADDR_SPACE #define SPIRV_RESERVE_ID_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE #endif // Excerpt from SPIR 2.0 spec.: // Pipe objects are represented using pointers to the opaque %opencl.pipe LLVM // structure type which reside in the global address space. #ifndef SPIRV_PIPE_ADDR_SPACE #define SPIRV_PIPE_ADDR_SPACE SPIRAS_Global #endif // Excerpt from SPIR 2.0 spec.: // Note: Images data types reside in global memory and hence should be marked // as such in the "kernel arg addr space" metadata. #ifndef SPIRV_IMAGE_ADDR_SPACE #define SPIRV_IMAGE_ADDR_SPACE SPIRAS_Global #endif } // namespace OCLUtil /////////////////////////////////////////////////////////////////////////////// // // Map definitions // /////////////////////////////////////////////////////////////////////////////// using namespace OCLUtil; namespace SPIRV { template <> void SPIRVMap::init() { add(OCLMF_Local, MemorySemanticsWorkgroupMemoryMask); add(OCLMF_Global, MemorySemanticsCrossWorkgroupMemoryMask); add(OCLMF_Image, MemorySemanticsImageMemoryMask); } template <> void SPIRVMap::init() { add(OCLMFEx_Local, MemorySemanticsWorkgroupMemoryMask); add(OCLMFEx_Global, MemorySemanticsCrossWorkgroupMemoryMask); add(OCLMFEx_Local_Global, MemorySemanticsWorkgroupMemoryMask | MemorySemanticsCrossWorkgroupMemoryMask); add(OCLMFEx_Image, MemorySemanticsImageMemoryMask); add(OCLMFEx_Image_Local, MemorySemanticsWorkgroupMemoryMask | MemorySemanticsImageMemoryMask); add(OCLMFEx_Image_Global, MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask); add(OCLMFEx_Image_Local_Global, MemorySemanticsWorkgroupMemoryMask | MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask); } template <> void SPIRVMap::init() { add(OCLMO_relaxed, MemorySemanticsMaskNone); add(OCLMO_acquire, MemorySemanticsAcquireMask); add(OCLMO_release, MemorySemanticsReleaseMask); add(OCLMO_acq_rel, MemorySemanticsAcquireReleaseMask); add(OCLMO_seq_cst, MemorySemanticsSequentiallyConsistentMask); } template <> void SPIRVMap::init() { add(OCLMS_work_item, ScopeInvocation); add(OCLMS_work_group, ScopeWorkgroup); add(OCLMS_device, ScopeDevice); add(OCLMS_all_svm_devices, ScopeCrossDevice); add(OCLMS_sub_group, ScopeSubgroup); } template <> void SPIRVMap::init() { add("reduce", GroupOperationReduce); add("scan_inclusive", GroupOperationInclusiveScan); add("scan_exclusive", GroupOperationExclusiveScan); add("ballot_bit_count", GroupOperationReduce); add("ballot_inclusive_scan", GroupOperationInclusiveScan); add("ballot_exclusive_scan", GroupOperationExclusiveScan); add("non_uniform_reduce", GroupOperationReduce); add("non_uniform_scan_inclusive", GroupOperationInclusiveScan); add("non_uniform_scan_exclusive", GroupOperationExclusiveScan); add("non_uniform_reduce_logical", GroupOperationReduce); add("non_uniform_scan_inclusive_logical", GroupOperationInclusiveScan); add("non_uniform_scan_exclusive_logical", GroupOperationExclusiveScan); add("clustered_reduce", GroupOperationClusteredReduce); } template <> void SPIRVMap::init() { add("rte", FPRoundingModeRTE); add("rtz", FPRoundingModeRTZ); add("rtp", FPRoundingModeRTP); add("rtn", FPRoundingModeRTN); } template <> void SPIRVMap::init() { #define _SPIRV_OP(x) add(OclExt::x, #x); _SPIRV_OP(cl_images) _SPIRV_OP(cl_doubles) _SPIRV_OP(cl_khr_int64_base_atomics) _SPIRV_OP(cl_khr_int64_extended_atomics) _SPIRV_OP(cl_khr_fp16) _SPIRV_OP(cl_khr_gl_sharing) _SPIRV_OP(cl_khr_gl_event) _SPIRV_OP(cl_khr_d3d10_sharing) _SPIRV_OP(cl_khr_media_sharing) _SPIRV_OP(cl_khr_d3d11_sharing) _SPIRV_OP(cl_khr_global_int32_base_atomics) _SPIRV_OP(cl_khr_global_int32_extended_atomics) _SPIRV_OP(cl_khr_local_int32_base_atomics) _SPIRV_OP(cl_khr_local_int32_extended_atomics) _SPIRV_OP(cl_khr_byte_addressable_store) _SPIRV_OP(cl_khr_3d_image_writes) _SPIRV_OP(cl_khr_gl_msaa_sharing) _SPIRV_OP(cl_khr_depth_images) _SPIRV_OP(cl_khr_gl_depth_images) _SPIRV_OP(cl_khr_subgroups) _SPIRV_OP(cl_khr_mipmap_image) _SPIRV_OP(cl_khr_mipmap_image_writes) _SPIRV_OP(cl_khr_egl_event) _SPIRV_OP(cl_khr_srgb_image_writes) _SPIRV_OP(cl_khr_extended_bit_ops) #undef _SPIRV_OP } template <> void SPIRVMap::init() { add(OclExt::cl_images, CapabilityImageBasic); add(OclExt::cl_doubles, CapabilityFloat64); add(OclExt::cl_khr_int64_base_atomics, CapabilityInt64Atomics); add(OclExt::cl_khr_int64_extended_atomics, CapabilityInt64Atomics); add(OclExt::cl_khr_fp16, CapabilityFloat16); add(OclExt::cl_khr_subgroups, CapabilityGroups); add(OclExt::cl_khr_mipmap_image, CapabilityImageMipmap); add(OclExt::cl_khr_mipmap_image_writes, CapabilityImageMipmap); add(OclExt::cl_khr_extended_bit_ops, CapabilityBitInstructions); } /// Map OpenCL work functions to SPIR-V builtin variables. template <> void SPIRVMap::init() { add("get_work_dim", BuiltInWorkDim); add("get_global_size", BuiltInGlobalSize); add("get_global_id", BuiltInGlobalInvocationId); add("get_global_offset", BuiltInGlobalOffset); add("get_local_size", BuiltInWorkgroupSize); add("get_enqueued_local_size", BuiltInEnqueuedWorkgroupSize); add("get_local_id", BuiltInLocalInvocationId); add("get_num_groups", BuiltInNumWorkgroups); add("get_group_id", BuiltInWorkgroupId); add("get_global_linear_id", BuiltInGlobalLinearId); add("get_local_linear_id", BuiltInLocalInvocationIndex); // cl_khr_subgroups add("get_sub_group_size", BuiltInSubgroupSize); add("get_max_sub_group_size", BuiltInSubgroupMaxSize); add("get_num_sub_groups", BuiltInNumSubgroups); add("get_enqueued_num_sub_groups", BuiltInNumEnqueuedSubgroups); add("get_sub_group_id", BuiltInSubgroupId); add("get_sub_group_local_id", BuiltInSubgroupLocalInvocationId); // cl_khr_subgroup_ballot add("get_sub_group_eq_mask", BuiltInSubgroupEqMask); add("get_sub_group_ge_mask", BuiltInSubgroupGeMask); add("get_sub_group_gt_mask", BuiltInSubgroupGtMask); add("get_sub_group_le_mask", BuiltInSubgroupLeMask); add("get_sub_group_lt_mask", BuiltInSubgroupLtMask); } // Maps uniqued OCL builtin function name to SPIR-V op code. // A uniqued OCL builtin function name may be different from the real // OCL builtin function name. e.g. instead of atomic_min, atomic_umin // is used for atomic_min with unsigned integer parameter. // work_group_ and sub_group_ functions are unified as group_ functions // except work_group_barrier. class SPIRVInstruction; template <> void SPIRVMap::init() { #define _SPIRV_OP(x, y) add("atom_" #x, OpAtomic##y); // cl_khr_int64_base_atomics builtins _SPIRV_OP(add, IAdd) _SPIRV_OP(sub, ISub) _SPIRV_OP(xchg, Exchange) _SPIRV_OP(dec, IDecrement) _SPIRV_OP(inc, IIncrement) _SPIRV_OP(cmpxchg, CompareExchange) // cl_khr_int64_extended_atomics builtins _SPIRV_OP(min, SMin) _SPIRV_OP(max, SMax) _SPIRV_OP(and, And) _SPIRV_OP(or, Or) _SPIRV_OP(xor, Xor) #undef _SPIRV_OP #define _SPIRV_OP(x, y) add("atomic_" #x, Op##y); // CL 2.0 atomic builtins _SPIRV_OP(flag_test_and_set_explicit, AtomicFlagTestAndSet) _SPIRV_OP(flag_clear_explicit, AtomicFlagClear) _SPIRV_OP(load_explicit, AtomicLoad) _SPIRV_OP(store_explicit, AtomicStore) _SPIRV_OP(exchange_explicit, AtomicExchange) _SPIRV_OP(compare_exchange_strong_explicit, AtomicCompareExchange) _SPIRV_OP(compare_exchange_weak_explicit, AtomicCompareExchangeWeak) _SPIRV_OP(inc, AtomicIIncrement) _SPIRV_OP(dec, AtomicIDecrement) _SPIRV_OP(fetch_add_explicit, AtomicIAdd) _SPIRV_OP(fetch_sub_explicit, AtomicISub) _SPIRV_OP(fetch_umin_explicit, AtomicUMin) _SPIRV_OP(fetch_umax_explicit, AtomicUMax) _SPIRV_OP(fetch_min_explicit, AtomicSMin) _SPIRV_OP(fetch_max_explicit, AtomicSMax) _SPIRV_OP(fetch_and_explicit, AtomicAnd) _SPIRV_OP(fetch_or_explicit, AtomicOr) _SPIRV_OP(fetch_xor_explicit, AtomicXor) #undef _SPIRV_OP #define _SPIRV_OP(x, y) add(#x, Op##y); _SPIRV_OP(dot, Dot) _SPIRV_OP(async_work_group_copy, GroupAsyncCopy) _SPIRV_OP(async_work_group_strided_copy, GroupAsyncCopy) _SPIRV_OP(wait_group_events, GroupWaitEvents) _SPIRV_OP(isequal, FOrdEqual) _SPIRV_OP(isnotequal, FUnordNotEqual) _SPIRV_OP(isgreater, FOrdGreaterThan) _SPIRV_OP(isgreaterequal, FOrdGreaterThanEqual) _SPIRV_OP(isless, FOrdLessThan) _SPIRV_OP(islessequal, FOrdLessThanEqual) _SPIRV_OP(islessgreater, FOrdNotEqual) _SPIRV_OP(isordered, Ordered) _SPIRV_OP(isunordered, Unordered) _SPIRV_OP(isfinite, IsFinite) _SPIRV_OP(isinf, IsInf) _SPIRV_OP(isnan, IsNan) _SPIRV_OP(isnormal, IsNormal) _SPIRV_OP(signbit, SignBitSet) _SPIRV_OP(any, Any) _SPIRV_OP(all, All) _SPIRV_OP(popcount, BitCount) _SPIRV_OP(get_fence, GenericPtrMemSemantics) // CL 2.0 kernel enqueue builtins _SPIRV_OP(enqueue_marker, EnqueueMarker) _SPIRV_OP(enqueue_kernel, EnqueueKernel) _SPIRV_OP(get_kernel_sub_group_count_for_ndrange_impl, GetKernelNDrangeSubGroupCount) _SPIRV_OP(get_kernel_max_sub_group_size_for_ndrange_impl, GetKernelNDrangeMaxSubGroupSize) _SPIRV_OP(get_kernel_work_group_size_impl, GetKernelWorkGroupSize) _SPIRV_OP(get_kernel_preferred_work_group_size_multiple_impl, GetKernelPreferredWorkGroupSizeMultiple) _SPIRV_OP(retain_event, RetainEvent) _SPIRV_OP(release_event, ReleaseEvent) _SPIRV_OP(create_user_event, CreateUserEvent) _SPIRV_OP(is_valid_event, IsValidEvent) _SPIRV_OP(set_user_event_status, SetUserEventStatus) _SPIRV_OP(capture_event_profiling_info, CaptureEventProfilingInfo) _SPIRV_OP(get_default_queue, GetDefaultQueue) _SPIRV_OP(ndrange_1D, BuildNDRange) _SPIRV_OP(ndrange_2D, BuildNDRange) _SPIRV_OP(ndrange_3D, BuildNDRange) // Generic Address Space Casts _SPIRV_OP(to_global, GenericCastToPtrExplicit) _SPIRV_OP(to_local, GenericCastToPtrExplicit) _SPIRV_OP(to_private, GenericCastToPtrExplicit) // CL 2.0 pipe builtins _SPIRV_OP(read_pipe_2, ReadPipe) _SPIRV_OP(write_pipe_2, WritePipe) _SPIRV_OP(read_pipe_2_bl, ReadPipeBlockingINTEL) _SPIRV_OP(write_pipe_2_bl, WritePipeBlockingINTEL) _SPIRV_OP(read_pipe_4, ReservedReadPipe) _SPIRV_OP(write_pipe_4, ReservedWritePipe) _SPIRV_OP(reserve_read_pipe, ReserveReadPipePackets) _SPIRV_OP(reserve_write_pipe, ReserveWritePipePackets) _SPIRV_OP(commit_read_pipe, CommitReadPipe) _SPIRV_OP(commit_write_pipe, CommitWritePipe) _SPIRV_OP(is_valid_reserve_id, IsValidReserveId) _SPIRV_OP(group_reserve_read_pipe, GroupReserveReadPipePackets) _SPIRV_OP(group_reserve_write_pipe, GroupReserveWritePipePackets) _SPIRV_OP(group_commit_read_pipe, GroupCommitReadPipe) _SPIRV_OP(group_commit_write_pipe, GroupCommitWritePipe) _SPIRV_OP(get_pipe_num_packets_ro, GetNumPipePackets) _SPIRV_OP(get_pipe_num_packets_wo, GetNumPipePackets) _SPIRV_OP(get_pipe_max_packets_ro, GetMaxPipePackets) _SPIRV_OP(get_pipe_max_packets_wo, GetMaxPipePackets) // CL 2.0 workgroup builtins _SPIRV_OP(group_all, GroupAll) _SPIRV_OP(group_any, GroupAny) _SPIRV_OP(group_broadcast, GroupBroadcast) _SPIRV_OP(group_iadd, GroupIAdd) _SPIRV_OP(group_fadd, GroupFAdd) _SPIRV_OP(group_fmin, GroupFMin) _SPIRV_OP(group_umin, GroupUMin) _SPIRV_OP(group_smin, GroupSMin) _SPIRV_OP(group_fmax, GroupFMax) _SPIRV_OP(group_umax, GroupUMax) _SPIRV_OP(group_smax, GroupSMax) _SPIRV_OP(group_imul, GroupIMulKHR) _SPIRV_OP(group_fmul, GroupFMulKHR) _SPIRV_OP(group_ibitwise_and, GroupBitwiseAndKHR) _SPIRV_OP(group_ibitwise_or, GroupBitwiseOrKHR) _SPIRV_OP(group_ibitwise_xor, GroupBitwiseXorKHR) _SPIRV_OP(group_ilogical_and, GroupLogicalAndKHR) _SPIRV_OP(group_ilogical_or, GroupLogicalOrKHR) _SPIRV_OP(group_ilogical_xor, GroupLogicalXorKHR) // CL image builtins _SPIRV_OP(SampledImage, SampledImage) _SPIRV_OP(ImageSampleExplicitLod, ImageSampleExplicitLod) _SPIRV_OP(read_image, ImageRead) _SPIRV_OP(write_image, ImageWrite) _SPIRV_OP(get_image_channel_data_type, ImageQueryFormat) _SPIRV_OP(get_image_channel_order, ImageQueryOrder) _SPIRV_OP(get_image_num_mip_levels, ImageQueryLevels) _SPIRV_OP(get_image_num_samples, ImageQuerySamples) // Intel Subgroups builtins _SPIRV_OP(intel_sub_group_shuffle, SubgroupShuffleINTEL) _SPIRV_OP(intel_sub_group_shuffle_down, SubgroupShuffleDownINTEL) _SPIRV_OP(intel_sub_group_shuffle_up, SubgroupShuffleUpINTEL) _SPIRV_OP(intel_sub_group_shuffle_xor, SubgroupShuffleXorINTEL) // Intel media_block_io builtins _SPIRV_OP(intel_sub_group_media_block_read, SubgroupImageMediaBlockReadINTEL) _SPIRV_OP(intel_sub_group_media_block_write, SubgroupImageMediaBlockWriteINTEL) // cl_khr_subgroup_non_uniform_vote _SPIRV_OP(group_elect, GroupNonUniformElect) _SPIRV_OP(group_non_uniform_all, GroupNonUniformAll) _SPIRV_OP(group_non_uniform_any, GroupNonUniformAny) _SPIRV_OP(group_non_uniform_all_equal, GroupNonUniformAllEqual) // cl_khr_subgroup_ballot _SPIRV_OP(group_non_uniform_broadcast, GroupNonUniformBroadcast) _SPIRV_OP(group_broadcast_first, GroupNonUniformBroadcastFirst) _SPIRV_OP(group_ballot, GroupNonUniformBallot) _SPIRV_OP(group_inverse_ballot, GroupNonUniformInverseBallot) _SPIRV_OP(group_ballot_bit_extract, GroupNonUniformBallotBitExtract) _SPIRV_OP(group_ballot_bit_count_iadd, GroupNonUniformBallotBitCount) _SPIRV_OP(group_ballot_find_lsb, GroupNonUniformBallotFindLSB) _SPIRV_OP(group_ballot_find_msb, GroupNonUniformBallotFindMSB) // cl_khr_subgroup_non_uniform_arithmetic _SPIRV_OP(group_non_uniform_iadd, GroupNonUniformIAdd) _SPIRV_OP(group_non_uniform_fadd, GroupNonUniformFAdd) _SPIRV_OP(group_non_uniform_imul, GroupNonUniformIMul) _SPIRV_OP(group_non_uniform_fmul, GroupNonUniformFMul) _SPIRV_OP(group_non_uniform_smin, GroupNonUniformSMin) _SPIRV_OP(group_non_uniform_umin, GroupNonUniformUMin) _SPIRV_OP(group_non_uniform_fmin, GroupNonUniformFMin) _SPIRV_OP(group_non_uniform_smax, GroupNonUniformSMax) _SPIRV_OP(group_non_uniform_umax, GroupNonUniformUMax) _SPIRV_OP(group_non_uniform_fmax, GroupNonUniformFMax) _SPIRV_OP(group_non_uniform_iand, GroupNonUniformBitwiseAnd) _SPIRV_OP(group_non_uniform_ior, GroupNonUniformBitwiseOr) _SPIRV_OP(group_non_uniform_ixor, GroupNonUniformBitwiseXor) _SPIRV_OP(group_non_uniform_logical_iand, GroupNonUniformLogicalAnd) _SPIRV_OP(group_non_uniform_logical_ior, GroupNonUniformLogicalOr) _SPIRV_OP(group_non_uniform_logical_ixor, GroupNonUniformLogicalXor) // cl_khr_subgroup_shuffle _SPIRV_OP(group_shuffle, GroupNonUniformShuffle) _SPIRV_OP(group_shuffle_xor, GroupNonUniformShuffleXor) // cl_khr_subgroup_shuffle_relative _SPIRV_OP(group_shuffle_up, GroupNonUniformShuffleUp) _SPIRV_OP(group_shuffle_down, GroupNonUniformShuffleDown) // cl_khr_subgroup_rotate _SPIRV_OP(group_rotate, GroupNonUniformRotateKHR) _SPIRV_OP(group_clustered_rotate, GroupNonUniformRotateKHR) // cl_khr_extended_bit_ops _SPIRV_OP(bitfield_insert, BitFieldInsert) _SPIRV_OP(bitfield_extract_signed, BitFieldSExtract) _SPIRV_OP(bitfield_extract_unsigned, BitFieldUExtract) _SPIRV_OP(bit_reverse, BitReverse) // cl_khr_split_work_group_barrier _SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL) _SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL) #undef _SPIRV_OP } template <> void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(#x, Op##y); _SPIRV_OP(add, AtomicIAdd) _SPIRV_OP(sub, AtomicISub) _SPIRV_OP(xchg, AtomicExchange) _SPIRV_OP(cmpxchg, AtomicCompareExchange) _SPIRV_OP(inc, AtomicIIncrement) _SPIRV_OP(dec, AtomicIDecrement) _SPIRV_OP(min, AtomicSMin) _SPIRV_OP(max, AtomicSMax) _SPIRV_OP(umin, AtomicUMin) _SPIRV_OP(umax, AtomicUMax) _SPIRV_OP(and, AtomicAnd) _SPIRV_OP(or, AtomicOr) _SPIRV_OP(xor, AtomicXor) #undef _SPIRV_OP } // SPV_INTEL_device_side_avc_motion_estimation extension builtins class SPIRVSubgroupsAVCIntelInst; template <> void SPIRVMap::init() { // Here is a workaround for a bug in the specification: // 'avc' missed in 'intel_sub_group_avc' prefix. add("intel_sub_group_ime_ref_window_size", OpSubgroupAvcImeRefWindowSizeINTEL); #define _SPIRV_OP(x, y) add("intel_sub_group_avc_" #x, OpSubgroupAvc##y##INTEL); // Initialization phase functions _SPIRV_OP(ime_initialize, ImeInitialize) _SPIRV_OP(fme_initialize, FmeInitialize) _SPIRV_OP(bme_initialize, BmeInitialize) _SPIRV_OP(sic_initialize, SicInitialize) // Result and payload types conversion functions _SPIRV_OP(mce_convert_to_ime_payload, MceConvertToImePayload) _SPIRV_OP(mce_convert_to_ime_result, MceConvertToImeResult) _SPIRV_OP(mce_convert_to_ref_payload, MceConvertToRefPayload) _SPIRV_OP(mce_convert_to_ref_result, MceConvertToRefResult) _SPIRV_OP(mce_convert_to_sic_payload, MceConvertToSicPayload) _SPIRV_OP(mce_convert_to_sic_result, MceConvertToSicResult) _SPIRV_OP(ime_convert_to_mce_payload, ImeConvertToMcePayload) _SPIRV_OP(ime_convert_to_mce_result, ImeConvertToMceResult) _SPIRV_OP(ref_convert_to_mce_payload, RefConvertToMcePayload) _SPIRV_OP(ref_convert_to_mce_result, RefConvertToMceResult) _SPIRV_OP(sic_convert_to_mce_payload, SicConvertToMcePayload) _SPIRV_OP(sic_convert_to_mce_result, SicConvertToMceResult) #undef _SPIRV_OP // MCE instructions #define _SPIRV_OP(x, y) \ add("intel_sub_group_avc_mce_" #x, OpSubgroupAvcMce##y##INTEL); _SPIRV_OP(get_default_inter_base_multi_reference_penalty, GetDefaultInterBaseMultiReferencePenalty) _SPIRV_OP(set_inter_base_multi_reference_penalty, SetInterBaseMultiReferencePenalty) _SPIRV_OP(get_default_inter_shape_penalty, GetDefaultInterShapePenalty) _SPIRV_OP(set_inter_shape_penalty, SetInterShapePenalty) _SPIRV_OP(get_default_inter_direction_penalty, GetDefaultInterDirectionPenalty) _SPIRV_OP(set_inter_direction_penalty, SetInterDirectionPenalty) _SPIRV_OP(get_default_intra_luma_shape_penalty, GetDefaultIntraLumaShapePenalty) _SPIRV_OP(get_default_inter_motion_vector_cost_table, GetDefaultInterMotionVectorCostTable) _SPIRV_OP(get_default_high_penalty_cost_table, GetDefaultHighPenaltyCostTable) _SPIRV_OP(get_default_medium_penalty_cost_table, GetDefaultMediumPenaltyCostTable) _SPIRV_OP(get_default_low_penalty_cost_table, GetDefaultLowPenaltyCostTable) _SPIRV_OP(set_motion_vector_cost_function, SetMotionVectorCostFunction) _SPIRV_OP(get_default_intra_luma_mode_penalty, GetDefaultIntraLumaModePenalty) _SPIRV_OP(get_default_non_dc_luma_intra_penalty, GetDefaultNonDcLumaIntraPenalty) _SPIRV_OP(get_default_intra_chroma_mode_base_penalty, GetDefaultIntraChromaModeBasePenalty) _SPIRV_OP(set_ac_only_haar, SetAcOnlyHaar) _SPIRV_OP(set_source_interlaced_field_polarity, SetSourceInterlacedFieldPolarity) _SPIRV_OP(set_single_reference_interlaced_field_polarity, SetSingleReferenceInterlacedFieldPolarity) _SPIRV_OP(set_dual_reference_interlaced_field_polarities, SetDualReferenceInterlacedFieldPolarities) _SPIRV_OP(get_motion_vectors, GetMotionVectors) _SPIRV_OP(get_inter_distortions, GetInterDistortions) _SPIRV_OP(get_best_inter_distortion, GetBestInterDistortions) _SPIRV_OP(get_inter_major_shape, GetInterMajorShape) _SPIRV_OP(get_inter_minor_shapes, GetInterMinorShape) _SPIRV_OP(get_inter_directions, GetInterDirections) _SPIRV_OP(get_inter_motion_vector_count, GetInterMotionVectorCount) _SPIRV_OP(get_inter_reference_ids, GetInterReferenceIds) _SPIRV_OP(get_inter_reference_interlaced_field_polarities, GetInterReferenceInterlacedFieldPolarities) #undef _SPIRV_OP // IME instructions #define _SPIRV_OP(x, y) \ add("intel_sub_group_avc_ime_" #x, OpSubgroupAvcIme##y##INTEL); _SPIRV_OP(set_single_reference, SetSingleReference) _SPIRV_OP(set_dual_reference, SetDualReference) _SPIRV_OP(ref_window_size, RefWindowSize) _SPIRV_OP(adjust_ref_offset, AdjustRefOffset) _SPIRV_OP(set_max_motion_vector_count, SetMaxMotionVectorCount) _SPIRV_OP(set_unidirectional_mix_disable, SetUnidirectionalMixDisable) _SPIRV_OP(set_early_search_termination_threshold, SetEarlySearchTerminationThreshold) _SPIRV_OP(set_weighted_sad, SetWeightedSad) _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference) _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference) _SPIRV_OP(evaluate_with_single_reference_streamin, EvaluateWithSingleReferenceStreamin) _SPIRV_OP(evaluate_with_dual_reference_streamin, EvaluateWithDualReferenceStreamin) _SPIRV_OP(evaluate_with_single_reference_streamout, EvaluateWithSingleReferenceStreamout) _SPIRV_OP(evaluate_with_dual_reference_streamout, EvaluateWithDualReferenceStreamout) _SPIRV_OP(evaluate_with_single_reference_streaminout, EvaluateWithSingleReferenceStreaminout) _SPIRV_OP(evaluate_with_dual_reference_streaminout, EvaluateWithDualReferenceStreaminout) _SPIRV_OP(get_single_reference_streamin, GetSingleReferenceStreamin) _SPIRV_OP(get_dual_reference_streamin, GetDualReferenceStreamin) _SPIRV_OP(strip_single_reference_streamout, StripSingleReferenceStreamout) _SPIRV_OP(strip_dual_reference_streamout, StripDualReferenceStreamout) _SPIRV_OP(get_border_reached, GetBorderReached) _SPIRV_OP(get_truncated_search_indication, GetTruncatedSearchIndication) _SPIRV_OP(get_unidirectional_early_search_termination, GetUnidirectionalEarlySearchTermination) _SPIRV_OP(get_weighting_pattern_minimum_motion_vector, GetWeightingPatternMinimumMotionVector) _SPIRV_OP(get_weighting_pattern_minimum_distortion, GetWeightingPatternMinimumDistortion) #undef _SPIRV_OP #define _SPIRV_OP(x, y) \ add("intel_sub_group_avc_ime_get_streamout_major_shape_" #x, \ OpSubgroupAvcImeGetStreamout##y##INTEL); _SPIRV_OP(motion_vectors_single_reference, SingleReferenceMajorShapeMotionVectors) _SPIRV_OP(distortions_single_reference, SingleReferenceMajorShapeDistortions) _SPIRV_OP(reference_ids_single_reference, SingleReferenceMajorShapeReferenceIds) _SPIRV_OP(motion_vectors_dual_reference, DualReferenceMajorShapeMotionVectors) _SPIRV_OP(distortions_dual_reference, DualReferenceMajorShapeDistortions) _SPIRV_OP(reference_ids_dual_reference, DualReferenceMajorShapeReferenceIds) #undef _SPIRV_OP // REF instructions #define _SPIRV_OP(x, y) \ add("intel_sub_group_avc_ref_" #x, OpSubgroupAvcRef##y##INTEL); _SPIRV_OP(set_bidirectional_mix_disable, SetBidirectionalMixDisable) _SPIRV_OP(set_bilinear_filter_enable, SetBilinearFilterEnable) _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference) _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference) _SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference) _SPIRV_OP(evaluate_with_multi_reference_interlaced, EvaluateWithMultiReferenceInterlaced) #undef _SPIRV_OP // SIC instructions #define _SPIRV_OP(x, y) \ add("intel_sub_group_avc_sic_" #x, OpSubgroupAvcSic##y##INTEL); _SPIRV_OP(configure_skc, ConfigureSkc) _SPIRV_OP(configure_ipe_luma, ConfigureIpeLuma) _SPIRV_OP(configure_ipe_luma_chroma, ConfigureIpeLumaChroma) _SPIRV_OP(get_motion_vector_mask, GetMotionVectorMask) _SPIRV_OP(set_intra_luma_shape_penalty, SetIntraLumaShapePenalty) _SPIRV_OP(set_intra_luma_mode_cost_function, SetIntraLumaModeCostFunction) _SPIRV_OP(set_intra_chroma_mode_cost_function, SetIntraChromaModeCostFunction) _SPIRV_OP(set_skc_bilinear_filter_enable, SetBilinearFilterEnable) _SPIRV_OP(set_skc_forward_transform_enable, SetSkcForwardTransformEnable) _SPIRV_OP(set_block_based_raw_skip_sad, SetBlockBasedRawSkipSad) _SPIRV_OP(evaluate_ipe, EvaluateIpe) _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference) _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference) _SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference) _SPIRV_OP(evaluate_with_multi_reference_interlaced, EvaluateWithMultiReferenceInterlaced) _SPIRV_OP(get_ipe_luma_shape, GetIpeLumaShape) _SPIRV_OP(get_best_ipe_luma_distortion, GetBestIpeLumaDistortion) _SPIRV_OP(get_best_ipe_chroma_distortion, GetBestIpeChromaDistortion) _SPIRV_OP(get_packed_ipe_luma_modes, GetPackedIpeLumaModes) _SPIRV_OP(get_ipe_chroma_mode, GetIpeChromaMode) _SPIRV_OP(get_packed_skc_luma_count_threshold, GetPackedSkcLumaCountThreshold) _SPIRV_OP(get_packed_skc_luma_sum_threshold, GetPackedSkcLumaSumThreshold) _SPIRV_OP(get_inter_raw_sads, GetInterRawSads) #undef _SPIRV_OP } template <> void SPIRVMap::init() { add("opencl.event_t", OpTypeEvent); add("opencl.pipe_t", OpTypePipe); add("opencl.clk_event_t", OpTypeDeviceEvent); add("opencl.reserve_id_t", OpTypeReserveId); add("opencl.queue_t", OpTypeQueue); add("opencl.sampler_t", OpTypeSampler); } template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() { add(llvm::AtomicRMWInst::Xchg, OpAtomicExchange); add(llvm::AtomicRMWInst::Add, OpAtomicIAdd); add(llvm::AtomicRMWInst::Sub, OpAtomicISub); add(llvm::AtomicRMWInst::And, OpAtomicAnd); add(llvm::AtomicRMWInst::Or, OpAtomicOr); add(llvm::AtomicRMWInst::Xor, OpAtomicXor); add(llvm::AtomicRMWInst::Max, OpAtomicSMax); add(llvm::AtomicRMWInst::Min, OpAtomicSMin); add(llvm::AtomicRMWInst::UMax, OpAtomicUMax); add(llvm::AtomicRMWInst::UMin, OpAtomicUMin); add(llvm::AtomicRMWInst::FAdd, OpAtomicFAddEXT); add(llvm::AtomicRMWInst::FMin, OpAtomicFMinEXT); add(llvm::AtomicRMWInst::FMax, OpAtomicFMaxEXT); } } // namespace SPIRV /////////////////////////////////////////////////////////////////////////////// // // Functions for getting builtin call info // /////////////////////////////////////////////////////////////////////////////// namespace OCLUtil { AtomicWorkItemFenceLiterals getAtomicWorkItemFenceLiterals(CallInst *CI) { return std::make_tuple(getArgAsInt(CI, 0), static_cast(getArgAsInt(CI, 1)), static_cast(getArgAsInt(CI, 2))); } size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name) { if (Name.startswith("atomic_compare_exchange")) return 2; return 1; } size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC) { if (OC == OpAtomicCompareExchange || OC == OpAtomicCompareExchangeWeak) return 2; return 1; } // atomic_fetch_[add, sub, min, max] and atomic_fetch_[add, sub, min, // max]_explicit functions declared in clang headers should be translated // to corresponding FP-typed Atomic Instructions bool isComputeAtomicOCLBuiltin(StringRef DemangledName) { if (!DemangledName.startswith(kOCLBuiltinName::AtomicPrefix) && !DemangledName.startswith(kOCLBuiltinName::AtomPrefix)) return false; return llvm::StringSwitch(DemangledName) .EndsWith("atomic_add", true) .EndsWith("atomic_sub", true) .EndsWith("atomic_min", true) .EndsWith("atomic_max", true) .EndsWith("atom_add", true) .EndsWith("atom_sub", true) .EndsWith("atom_min", true) .EndsWith("atom_max", true) .EndsWith("inc", true) .EndsWith("dec", true) .EndsWith("cmpxchg", true) .EndsWith("and", true) .EndsWith("or", true) .EndsWith("xor", true) .EndsWith("or_explicit", true) .EndsWith("xor_explicit", true) .EndsWith("and_explicit", true) .Default(false); } BarrierLiterals getBarrierLiterals(CallInst *CI) { auto N = CI->arg_size(); assert(N == 1 || N == 2); StringRef DemangledName; assert(CI->getCalledFunction() && "Unexpected indirect call"); if (!oclIsBuiltin(CI->getCalledFunction()->getName(), DemangledName)) { assert(0 && "call must a builtin (work_group_barrier or sub_group_barrier)"); } OCLScopeKind Scope = OCLMS_work_group; if (DemangledName == kOCLBuiltinName::SubGroupBarrier) { Scope = OCLMS_sub_group; } return std::make_tuple(getArgAsInt(CI, 0), N == 1 ? OCLMS_work_group : static_cast(getArgAsInt(CI, 1)), Scope); } unsigned getExtOp(StringRef OrigName, StringRef GivenDemangledName) { std::string DemangledName{GivenDemangledName}; if (DemangledName.empty() || !oclIsBuiltin(OrigName, GivenDemangledName)) return ~0U; LLVM_DEBUG(dbgs() << "getExtOp: demangled name: " << DemangledName << '\n'); OCLExtOpKind EOC; bool Found = OCLExtOpMap::rfind(DemangledName, &EOC); if (!Found) { std::string Prefix; switch (lastFuncParamType(OrigName)) { case ParamType::UNSIGNED: Prefix = "u_"; break; case ParamType::SIGNED: Prefix = "s_"; break; case ParamType::FLOAT: Prefix = "f"; break; case ParamType::UNKNOWN: break; } Found = OCLExtOpMap::rfind(Prefix + DemangledName, &EOC); } if (Found) return EOC; else return ~0U; } /////////////////////////////////////////////////////////////////////////////// // // Functions for getting module info // /////////////////////////////////////////////////////////////////////////////// unsigned encodeOCLVer(unsigned short Major, unsigned char Minor, unsigned char Rev) { return (Major * 100 + Minor) * 1000 + Rev; } std::tuple decodeOCLVer(unsigned Ver) { unsigned short Major = Ver / 100000; unsigned char Minor = (Ver % 100000) / 1000; unsigned char Rev = Ver % 1000; return std::make_tuple(Major, Minor, Rev); } unsigned getOCLVersion(Module *M, bool AllowMulti) { NamedMDNode *NamedMD = M->getNamedMetadata(kSPIR2MD::OCLVer); if (!NamedMD) return 0; assert(NamedMD->getNumOperands() > 0 && "Invalid SPIR"); if (!AllowMulti && NamedMD->getNumOperands() != 1) report_fatal_error( llvm::Twine("Multiple OCL version metadata not allowed")); // If the module was linked with another module, there may be multiple // operands. auto GetVer = [=](unsigned I) { auto MD = NamedMD->getOperand(I); return std::make_pair(getMDOperandAsInt(MD, 0), getMDOperandAsInt(MD, 1)); }; auto Ver = GetVer(0); for (unsigned I = 1, E = NamedMD->getNumOperands(); I != E; ++I) if (Ver != GetVer(I)) report_fatal_error(llvm::Twine("OCL version mismatch")); return encodeOCLVer(Ver.first, Ver.second, 0); } void decodeMDNode(MDNode *N, unsigned &X, unsigned &Y, unsigned &Z) { if (N == NULL) return; X = getMDOperandAsInt(N, 0); Y = getMDOperandAsInt(N, 1); Z = getMDOperandAsInt(N, 2); } /// Encode LLVM type by SPIR-V execution mode VecTypeHint unsigned encodeVecTypeHint(Type *Ty) { if (Ty->isHalfTy()) return 4; if (Ty->isFloatTy()) return 5; if (Ty->isDoubleTy()) return 6; if (IntegerType *IntTy = dyn_cast(Ty)) { switch (IntTy->getIntegerBitWidth()) { case 8: return 0; case 16: return 1; case 32: return 2; case 64: return 3; default: llvm_unreachable("invalid integer type"); } } if (FixedVectorType *VecTy = dyn_cast(Ty)) { Type *EleTy = VecTy->getElementType(); unsigned Size = VecTy->getNumElements(); return Size << 16 | encodeVecTypeHint(EleTy); } llvm_unreachable("invalid type"); return ~0U; } Type *decodeVecTypeHint(LLVMContext &C, unsigned Code) { unsigned VecWidth = Code >> 16; unsigned Scalar = Code & 0xFFFF; Type *ST = nullptr; switch (Scalar) { case 0: case 1: case 2: case 3: ST = IntegerType::get(C, 1 << (3 + Scalar)); break; case 4: ST = Type::getHalfTy(C); break; case 5: ST = Type::getFloatTy(C); break; case 6: ST = Type::getDoubleTy(C); break; default: llvm_unreachable("Invalid vec type hint"); return nullptr; } if (VecWidth < 1) return ST; return FixedVectorType::get(ST, VecWidth); } unsigned transVecTypeHint(MDNode *Node) { return encodeVecTypeHint(getMDOperandAsType(Node, 0)); } SPIRAddressSpace getOCLOpaqueTypeAddrSpace(Op OpCode) { switch (OpCode) { case OpTypeQueue: return SPIRV_QUEUE_T_ADDR_SPACE; case OpTypeEvent: return SPIRV_EVENT_T_ADDR_SPACE; case OpTypeDeviceEvent: return SPIRV_CLK_EVENT_T_ADDR_SPACE; case OpTypeReserveId: return SPIRV_RESERVE_ID_T_ADDR_SPACE; case OpTypePipe: case OpTypePipeStorage: return SPIRV_PIPE_ADDR_SPACE; case OpTypeImage: case OpTypeSampledImage: return SPIRV_IMAGE_ADDR_SPACE; case OpConstantSampler: case OpTypeSampler: return SPIRV_SAMPLER_T_ADDR_SPACE; default: if (isSubgroupAvcINTELTypeOpCode(OpCode)) return SPIRV_AVC_INTEL_T_ADDR_SPACE; assert(false && "No address space is determined for some OCL type"); return SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE; } } static SPIR::TypeAttributeEnum mapAddrSpaceEnums(SPIRAddressSpace Addrspace) { switch (Addrspace) { case SPIRAS_Private: return SPIR::ATTR_PRIVATE; case SPIRAS_Global: return SPIR::ATTR_GLOBAL; case SPIRAS_Constant: return SPIR::ATTR_CONSTANT; case SPIRAS_Local: return SPIR::ATTR_LOCAL; case SPIRAS_Generic: return SPIR::ATTR_GENERIC; case SPIRAS_GlobalDevice: return SPIR::ATTR_GLOBAL_DEVICE; case SPIRAS_GlobalHost: return SPIR::ATTR_GLOBAL_HOST; default: llvm_unreachable("Invalid addrspace enum member"); } return SPIR::ATTR_NONE; } SPIR::TypeAttributeEnum getOCLOpaqueTypeAddrSpace(SPIR::TypePrimitiveEnum Prim) { switch (Prim) { case SPIR::PRIMITIVE_QUEUE_T: return mapAddrSpaceEnums(SPIRV_QUEUE_T_ADDR_SPACE); case SPIR::PRIMITIVE_EVENT_T: return mapAddrSpaceEnums(SPIRV_EVENT_T_ADDR_SPACE); case SPIR::PRIMITIVE_CLK_EVENT_T: return mapAddrSpaceEnums(SPIRV_CLK_EVENT_T_ADDR_SPACE); case SPIR::PRIMITIVE_RESERVE_ID_T: return mapAddrSpaceEnums(SPIRV_RESERVE_ID_T_ADDR_SPACE); case SPIR::PRIMITIVE_PIPE_RO_T: case SPIR::PRIMITIVE_PIPE_WO_T: return mapAddrSpaceEnums(SPIRV_PIPE_ADDR_SPACE); case SPIR::PRIMITIVE_IMAGE1D_RO_T: case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RO_T: case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RO_T: case SPIR::PRIMITIVE_IMAGE2D_RO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RO_T: case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_RO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T: case SPIR::PRIMITIVE_IMAGE3D_RO_T: case SPIR::PRIMITIVE_IMAGE1D_WO_T: case SPIR::PRIMITIVE_IMAGE1D_ARRAY_WO_T: case SPIR::PRIMITIVE_IMAGE1D_BUFFER_WO_T: case SPIR::PRIMITIVE_IMAGE2D_WO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_WO_T: case SPIR::PRIMITIVE_IMAGE2D_DEPTH_WO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_WO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T: case SPIR::PRIMITIVE_IMAGE3D_WO_T: case SPIR::PRIMITIVE_IMAGE1D_RW_T: case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RW_T: case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RW_T: case SPIR::PRIMITIVE_IMAGE2D_RW_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RW_T: case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RW_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_RW_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T: case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T: case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T: case SPIR::PRIMITIVE_IMAGE3D_RW_T: return mapAddrSpaceEnums(SPIRV_IMAGE_ADDR_SPACE); default: llvm_unreachable("No address space is determined for a SPIR primitive"); } return SPIR::ATTR_NONE; } // Fetch type of invoke function passed to device execution built-ins static FunctionType *getBlockInvokeTy(Function *F, unsigned BlockIdx) { auto Params = F->getFunctionType()->params(); PointerType *FuncPtr = cast(Params[BlockIdx]); return FunctionType::get(FuncPtr, Params, false); } class OCLBuiltinFuncMangleInfo : public SPIRV::BuiltinFuncMangleInfo { public: OCLBuiltinFuncMangleInfo(Function *F) : F(F) {} OCLBuiltinFuncMangleInfo(ArrayRef ArgTypes) : ArgTypes(ArgTypes.vec()) {} Type *getArgTy(unsigned I) { return F->getFunctionType()->getParamType(I); } void init(StringRef UniqName) override { // Make a local copy as we will modify the string in init function std::string TempStorage = UniqName.str(); auto NameRef = StringRef(TempStorage); // Helper functions to erase substrings from NameRef (i.e. TempStorage) auto EraseSubstring = [&NameRef, &TempStorage](const std::string &ToErase) { size_t Pos = TempStorage.find(ToErase); if (Pos != std::string::npos) { TempStorage.erase(Pos, ToErase.length()); // re-take StringRef as TempStorage was updated NameRef = StringRef(TempStorage); } }; auto EraseSymbol = [&NameRef, &TempStorage](size_t Index) { TempStorage.erase(Index, 1); // re-take StringRef as TempStorage was updated NameRef = StringRef(TempStorage); }; if (NameRef.startswith("async_work_group")) { addUnsignedArg(-1); setArgAttr(1, SPIR::ATTR_CONST); } else if (NameRef.startswith("printf")) setVarArg(1); else if (NameRef.startswith("write_imageui")) addUnsignedArg(2); else if (NameRef.equals("prefetch")) { addUnsignedArg(1); setArgAttr(0, SPIR::ATTR_CONST); } else if (NameRef.equals("get_kernel_work_group_size") || NameRef.equals( "get_kernel_preferred_work_group_size_multiple")) { assert(F && "lack of necessary information"); const size_t BlockArgIdx = 0; FunctionType *InvokeTy = getBlockInvokeTy(F, BlockArgIdx); if (InvokeTy->getNumParams() > 1) setLocalArgBlock(BlockArgIdx); } else if (NameRef.startswith("__enqueue_kernel")) { // clang doesn't mangle enqueue_kernel builtins setAsDontMangle(); } else if (NameRef.startswith("get_") || NameRef.equals("nan") || NameRef.equals("mem_fence") || NameRef.startswith("shuffle")) { addUnsignedArg(-1); if (NameRef.startswith(kOCLBuiltinName::GetFence)) { setArgAttr(0, SPIR::ATTR_CONST); addVoidPtrArg(0); } } else if (NameRef.contains("barrier")) { addUnsignedArg(0); if (NameRef.equals("work_group_barrier") || NameRef.equals("sub_group_barrier") || NameRef.equals("intel_work_group_barrier_arrive") || NameRef.equals("intel_work_group_barrier_wait")) setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE); } else if (NameRef.startswith("atomic_work_item_fence")) { addUnsignedArg(0); setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER); setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE); } else if (NameRef.startswith("atom_")) { setArgAttr(0, SPIR::ATTR_VOLATILE); if (NameRef.endswith("_umax") || NameRef.endswith("_umin")) { addUnsignedArg(-1); // We need to remove u to match OpenCL C built-in function name EraseSymbol(5); } } else if (NameRef.startswith("atomic")) { setArgAttr(0, SPIR::ATTR_VOLATILE); if (NameRef.contains("_umax") || NameRef.contains("_umin")) { addUnsignedArg(-1); // We need to remove u to match OpenCL C built-in function name if (NameRef.contains("_fetch")) EraseSymbol(13); else EraseSymbol(7); } if (NameRef.contains("store_explicit") || NameRef.contains("exchange_explicit") || (NameRef.startswith("atomic_fetch") && NameRef.contains("explicit"))) { setEnumArg(2, SPIR::PRIMITIVE_MEMORY_ORDER); setEnumArg(3, SPIR::PRIMITIVE_MEMORY_SCOPE); } else if (NameRef.contains("load_explicit") || (NameRef.startswith("atomic_flag") && NameRef.contains("explicit"))) { setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER); setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE); } else if (NameRef.endswith("compare_exchange_strong_explicit") || NameRef.endswith("compare_exchange_weak_explicit")) { setEnumArg(3, SPIR::PRIMITIVE_MEMORY_ORDER); setEnumArg(4, SPIR::PRIMITIVE_MEMORY_ORDER); setEnumArg(5, SPIR::PRIMITIVE_MEMORY_SCOPE); } // Don't set atomic property to the first argument of 1.2 atomic // built-ins. if (!NameRef.endswith("xchg") && // covers _cmpxchg too (NameRef.contains("fetch") || !(NameRef.endswith("_add") || NameRef.endswith("_sub") || NameRef.endswith("_inc") || NameRef.endswith("_dec") || NameRef.endswith("_min") || NameRef.endswith("_max") || NameRef.endswith("_and") || NameRef.endswith("_or") || NameRef.endswith("_xor")))) { addAtomicArg(0); } } else if (NameRef.startswith("uconvert_")) { addUnsignedArg(0); NameRef = NameRef.drop_front(1); UnmangledName.erase(0, 1); } else if (NameRef.startswith("s_")) { if (NameRef.equals("s_upsample")) addUnsignedArg(1); NameRef = NameRef.drop_front(2); } else if (NameRef.startswith("u_")) { addUnsignedArg(-1); NameRef = NameRef.drop_front(2); } else if (NameRef.equals("fclamp")) { NameRef = NameRef.drop_front(1); } // handle [read|write]pipe builtins (plus two i32 literal args // required by SPIR 2.0 provisional specification): else if (NameRef.equals("read_pipe_2") || NameRef.equals("write_pipe_2")) { // with 2 arguments (plus two i32 literals): // int read_pipe (read_only pipe gentype p, gentype *ptr) // int write_pipe (write_only pipe gentype p, const gentype *ptr) addVoidPtrArg(1); addUnsignedArg(2); addUnsignedArg(3); // OpenCL-like representation of blocking pipes } else if (NameRef.equals("read_pipe_2_bl") || NameRef.equals("write_pipe_2_bl")) { // with 2 arguments (plus two i32 literals): // int read_pipe_bl (read_only pipe gentype p, gentype *ptr) // int write_pipe_bl (write_only pipe gentype p, const gentype *ptr) addVoidPtrArg(1); addUnsignedArg(2); addUnsignedArg(3); } else if (NameRef.equals("read_pipe_4") || NameRef.equals("write_pipe_4")) { // with 4 arguments (plus two i32 literals): // int read_pipe (read_only pipe gentype p, reserve_id_t reserve_id, uint // index, gentype *ptr) int write_pipe (write_only pipe gentype p, // reserve_id_t reserve_id, uint index, const gentype *ptr) addUnsignedArg(2); addVoidPtrArg(3); addUnsignedArg(4); addUnsignedArg(5); } else if (NameRef.contains("reserve_read_pipe") || NameRef.contains("reserve_write_pipe")) { // process [|work_group|sub_group]reserve[read|write]pipe builtins addUnsignedArg(1); addUnsignedArg(2); addUnsignedArg(3); } else if (NameRef.contains("commit_read_pipe") || NameRef.contains("commit_write_pipe")) { // process [|work_group|sub_group]commit[read|write]pipe builtins addUnsignedArg(2); addUnsignedArg(3); } else if (NameRef.equals("capture_event_profiling_info")) { addVoidPtrArg(2); setEnumArg(1, SPIR::PRIMITIVE_CLK_PROFILING_INFO); } else if (NameRef.equals("enqueue_marker")) { setArgAttr(2, SPIR::ATTR_CONST); addUnsignedArg(1); } else if (NameRef.startswith("vload")) { addUnsignedArg(0); setArgAttr(1, SPIR::ATTR_CONST); } else if (NameRef.startswith("vstore")) { addUnsignedArg(1); } else if (NameRef.startswith("ndrange_")) { addUnsignedArgs(0, 2); if (NameRef[8] == '2' || NameRef[8] == '3') { setArgAttr(0, SPIR::ATTR_CONST); setArgAttr(1, SPIR::ATTR_CONST); setArgAttr(2, SPIR::ATTR_CONST); } } else if (NameRef.contains("umax")) { addUnsignedArg(-1); EraseSymbol(NameRef.find("umax")); } else if (NameRef.contains("umin")) { addUnsignedArg(-1); EraseSymbol(NameRef.find("umin")); } else if (NameRef.contains("broadcast")) { addUnsignedArg(-1); } else if (NameRef.startswith(kOCLBuiltinName::SampledReadImage)) { NameRef.consume_front(kOCLBuiltinName::Sampled); addSamplerArg(1); } else if (NameRef.contains(kOCLSubgroupsAVCIntel::Prefix)) { if (NameRef.contains("evaluate_ipe")) addSamplerArg(1); else if (NameRef.contains("evaluate_with_single_reference")) addSamplerArg(2); else if (NameRef.contains("evaluate_with_multi_reference")) { addUnsignedArg(1); std::string PostFix = "_interlaced"; if (NameRef.contains(PostFix)) { addUnsignedArg(2); addSamplerArg(3); EraseSubstring(PostFix); } else addSamplerArg(2); } else if (NameRef.contains("evaluate_with_dual_reference")) addSamplerArg(3); else if (NameRef.contains("fme_initialize")) addUnsignedArgs(0, 6); else if (NameRef.contains("bme_initialize")) addUnsignedArgs(0, 7); else if (NameRef.contains("set_inter_base_multi_reference_penalty") || NameRef.contains("set_inter_shape_penalty") || NameRef.contains("set_inter_direction_penalty")) addUnsignedArg(0); else if (NameRef.contains("set_motion_vector_cost_function")) addUnsignedArgs(0, 2); else if (NameRef.contains("interlaced_field_polarity")) addUnsignedArg(0); else if (NameRef.contains("interlaced_field_polarities")) addUnsignedArgs(0, 1); else if (NameRef.contains(kOCLSubgroupsAVCIntel::MCEPrefix)) { if (NameRef.contains("get_default")) addUnsignedArgs(0, 1); } else if (NameRef.contains(kOCLSubgroupsAVCIntel::IMEPrefix)) { if (NameRef.contains("initialize")) addUnsignedArgs(0, 2); else if (NameRef.contains("set_single_reference")) addUnsignedArg(1); else if (NameRef.contains("set_dual_reference")) addUnsignedArg(2); else if (NameRef.contains("set_weighted_sad") || NameRef.contains("set_early_search_termination_threshold")) addUnsignedArg(0); else if (NameRef.contains("adjust_ref_offset")) addUnsignedArgs(1, 3); else if (NameRef.contains("set_max_motion_vector_count") || NameRef.contains("get_border_reached")) addUnsignedArg(0); else if (NameRef.contains("shape_distortions") || NameRef.contains("shape_motion_vectors") || NameRef.contains("shape_reference_ids")) { if (NameRef.contains("single_reference")) { addUnsignedArg(1); EraseSubstring("_single_reference"); } else if (NameRef.contains("dual_reference")) { addUnsignedArgs(1, 2); EraseSubstring("_dual_reference"); } } else if (NameRef.contains("ref_window_size")) addUnsignedArg(0); } else if (NameRef.contains(kOCLSubgroupsAVCIntel::SICPrefix)) { if (NameRef.contains("initialize") || NameRef.contains("set_intra_luma_shape_penalty")) addUnsignedArg(0); else if (NameRef.contains("configure_ipe")) { if (NameRef.contains("_luma")) { addUnsignedArgs(0, 6); EraseSubstring("_luma"); } if (NameRef.contains("_chroma")) { addUnsignedArgs(7, 9); EraseSubstring("_chroma"); } } else if (NameRef.contains("configure_skc")) addUnsignedArgs(0, 4); else if (NameRef.contains("set_skc")) { if (NameRef.contains("forward_transform_enable")) addUnsignedArg(0); } else if (NameRef.contains("set_block")) { if (NameRef.contains("based_raw_skip_sad")) addUnsignedArg(0); } else if (NameRef.contains("get_motion_vector_mask")) { addUnsignedArgs(0, 1); } else if (NameRef.contains("luma_mode_cost_function")) addUnsignedArgs(0, 2); else if (NameRef.contains("chroma_mode_cost_function")) addUnsignedArg(0); } } else if (NameRef.startswith("intel_sub_group_shuffle")) { if (NameRef.endswith("_down") || NameRef.endswith("_up")) addUnsignedArg(2); else addUnsignedArg(1); } else if (NameRef.startswith("intel_sub_group_block_write")) { // distinguish write to image and other data types based on number of // arguments--images have one more argument. if (F->getFunctionType()->getNumParams() == 2) { addUnsignedArg(0); addUnsignedArg(1); } else { addUnsignedArg(2); } } else if (NameRef.startswith("intel_sub_group_block_read")) { // distinguish read from image and other data types based on number of // arguments--images have one more argument. if (F->getFunctionType()->getNumParams() == 1) { setArgAttr(0, SPIR::ATTR_CONST); addUnsignedArg(0); } } else if (NameRef.startswith("intel_sub_group_media_block_write")) { addUnsignedArg(3); } else if (NameRef.startswith(kOCLBuiltinName::SubGroupPrefix)) { if (NameRef.contains("ballot")) { if (NameRef.contains("inverse") || NameRef.contains("bit_count") || NameRef.contains("inclusive_scan") || NameRef.contains("exclusive_scan") || NameRef.contains("find_lsb") || NameRef.contains("find_msb")) addUnsignedArg(0); else if (NameRef.contains("bit_extract")) { addUnsignedArgs(0, 1); } } else if (NameRef.startswith("sub_group_clustered_rotate")) { addUnsignedArg(2); } else if (NameRef.contains("shuffle") || NameRef.contains("clustered")) addUnsignedArg(1); } else if (NameRef.startswith("bitfield_insert")) { addUnsignedArgs(2, 3); } else if (NameRef.startswith("bitfield_extract_signed") || NameRef.startswith("bitfield_extract_unsigned")) { addUnsignedArgs(1, 2); } // Store the final version of a function name UnmangledName = NameRef.str(); } // Auxiliarry information, it is expected that it is relevant at the moment // the init method is called. Function *F; // SPIRV decorated function // TODO: ArgTypes argument should get removed once all SPV-IR related issues // are resolved std::vector ArgTypes; // Arguments of OCL builtin }; CallInst *mutateCallInstOCL( Module *M, CallInst *CI, std::function &)> ArgMutate, AttributeList *Attrs) { OCLBuiltinFuncMangleInfo BtnInfo(CI->getCalledFunction()); return mutateCallInst(M, CI, ArgMutate, &BtnInfo, Attrs); } Instruction *mutateCallInstOCL( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, AttributeList *Attrs, bool TakeFuncName) { OCLBuiltinFuncMangleInfo BtnInfo(CI->getCalledFunction()); return mutateCallInst(M, CI, ArgMutate, RetMutate, &BtnInfo, Attrs, TakeFuncName); } static StringRef getStructName(Type *Ty) { if (auto *STy = dyn_cast(Ty)) return STy->isLiteral() ? "" : Ty->getStructName(); return ""; } Value *unwrapSpecialTypeInitializer(Value *V) { if (auto *BC = dyn_cast(V)) { Type *DestTy = BC->getDestTy(); Type *SrcTy = BC->getSrcTy(); if (SrcTy->isPointerTy() && !SrcTy->isOpaquePointerTy()) { StringRef SrcName = getStructName(SrcTy->getNonOpaquePointerElementType()); StringRef DestName = getStructName(DestTy->getNonOpaquePointerElementType()); if (DestName == getSPIRVTypeName(kSPIRVTypeName::PipeStorage) && SrcName == getSPIRVTypeName(kSPIRVTypeName::ConstantPipeStorage)) return BC->getOperand(0); if (DestName == getSPIRVTypeName(kSPIRVTypeName::Sampler) && SrcName == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler)) return BC->getOperand(0); } } return nullptr; } bool isSamplerStructTy(StructType *STy) { return STy && STy->hasName() && STy->getName() == kSPR2TypeName::Sampler; } bool isPipeOrAddressSpaceCastBI(const StringRef MangledName) { return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" || MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" || MangledName == "write_pipe_4" || MangledName == "read_pipe_4" || MangledName == "reserve_write_pipe" || MangledName == "reserve_read_pipe" || MangledName == "commit_write_pipe" || MangledName == "commit_read_pipe" || MangledName == "work_group_reserve_write_pipe" || MangledName == "work_group_reserve_read_pipe" || MangledName == "work_group_commit_write_pipe" || MangledName == "work_group_commit_read_pipe" || MangledName == "get_pipe_num_packets_ro" || MangledName == "get_pipe_max_packets_ro" || MangledName == "get_pipe_num_packets_wo" || MangledName == "get_pipe_max_packets_wo" || MangledName == "sub_group_reserve_write_pipe" || MangledName == "sub_group_reserve_read_pipe" || MangledName == "sub_group_commit_write_pipe" || MangledName == "sub_group_commit_read_pipe" || MangledName == "to_global" || MangledName == "to_local" || MangledName == "to_private"; } bool isEnqueueKernelBI(const StringRef MangledName) { return MangledName == "__enqueue_kernel_basic" || MangledName == "__enqueue_kernel_basic_events" || MangledName == "__enqueue_kernel_varargs" || MangledName == "__enqueue_kernel_events_varargs"; } bool isKernelQueryBI(const StringRef MangledName) { return MangledName == "__get_kernel_work_group_size_impl" || MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" || MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" || MangledName == "__get_kernel_preferred_work_group_size_multiple_impl"; } // isUnfusedMulAdd checks if we have the following (most common for fp // contranction) pattern in LLVM IR: // // %mul = fmul float %a, %b // %add = fadd float %mul, %c // // This pattern indicates that fp contraction could have been disabled by // #pragma OPENCL FP_CONTRACT OFF. When contraction is enabled (by a pragma or // by clang's -ffp-contract=fast), clang would generate: // // %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) // // or // // %mul = fmul contract float %a, %b // %add = fadd contract float %mul, %c // // Note that optimizations may form an unfused fmuladd from fadd+load or // fadd+call, so this check is quite restrictive (see the comment below). // bool isUnfusedMulAdd(BinaryOperator *B) { if (B->getOpcode() != Instruction::FAdd && B->getOpcode() != Instruction::FSub) return false; if (B->hasAllowContract()) { // If this fadd or fsub itself has a contract flag, the operation can be // contracted regardless of the operands. return false; } // Otherwise, we cannot easily tell if the operation can be a candidate for // contraction or not. Consider the following cases: // // %mul = alloca float // %t1 = fmul float %a, %b // store float* %mul, float %t // %t2 = load %mul // %r = fadd float %t2, %c // // LLVM IR does not allow %r to be contracted. However, after an optimization // it becomes a candidate for contraction if ContractionOFF is not set in // SPIR-V: // // %t1 = fmul float %a, %b // %r = fadd float %t1, %c // // To be on a safe side, we disallow everything that is even remotely similar // to fmul + fadd. return true; } std::string getIntelSubgroupBlockDataPostfix(unsigned ElementBitSize, unsigned VectorNumElements) { std::ostringstream OSS; switch (ElementBitSize) { case 8: OSS << "_uc"; break; case 16: OSS << "_us"; break; case 32: // Intentionally does nothing since _ui variant is only an alias. break; case 64: OSS << "_ul"; break; default: llvm_unreachable( "Incorrect data bitsize for intel_subgroup_block builtins"); } switch (VectorNumElements) { case 1: break; case 2: case 4: case 8: OSS << VectorNumElements; break; case 16: assert((ElementBitSize == 8 || ElementBitSize == 16) && "16 elements vector allowed only for char and short builtins"); OSS << VectorNumElements; break; default: llvm_unreachable( "Incorrect vector length for intel_subgroup_block builtins"); } return OSS.str(); } void insertImageNameAccessQualifier(SPIRVAccessQualifierKind Acc, std::string &Name) { std::string QName = rmap(Acc); // transform: read_only -> ro, write_only -> wo, read_write -> rw QName = QName.substr(0, 1) + QName.substr(QName.find("_") + 1, 1) + "_"; assert(!Name.empty() && "image name should not be empty"); Name.insert(Name.size() - 1, QName); } } // namespace OCLUtil Value *SPIRV::transOCLMemScopeIntoSPIRVScope(Value *MemScope, Optional DefaultCase, Instruction *InsertBefore) { if (auto *C = dyn_cast(MemScope)) { return ConstantInt::get( C->getType(), map(static_cast(C->getZExtValue()))); } // If memory_scope is not a constant, then we have to insert dynamic mapping: return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemScope, MemScope, OCLMemScopeMap::getMap(), /* IsReverse */ false, DefaultCase, InsertBefore); } Value *SPIRV::transOCLMemOrderIntoSPIRVMemorySemantics( Value *MemOrder, Optional DefaultCase, Instruction *InsertBefore) { if (auto *C = dyn_cast(MemOrder)) { return ConstantInt::get( C->getType(), mapOCLMemSemanticToSPIRV( 0, static_cast(C->getZExtValue()))); } return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemOrder, MemOrder, OCLMemOrderMap::getMap(), /* IsReverse */ false, DefaultCase, InsertBefore); } Value * SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Value *MemScope, Instruction *InsertBefore) { if (auto *C = dyn_cast(MemScope)) { return ConstantInt::get(C->getType(), rmap(static_cast( C->getZExtValue()))); } if (auto *CI = dyn_cast(MemScope)) { Function *F = CI->getCalledFunction(); if (F && F->getName().equals(kSPIRVName::TranslateOCLMemScope)) { // In case the SPIR-V module was created from an OpenCL program by // *this* SPIR-V generator, we know that the value passed to // __translate_ocl_memory_scope is what we should pass to the // OpenCL builtin now. return CI->getArgOperand(0); } } return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemScope, MemScope, OCLMemScopeMap::getRMap(), /* IsReverse */ true, None, InsertBefore); } Value * SPIRV::transSPIRVMemorySemanticsIntoOCLMemoryOrder(Value *MemorySemantics, Instruction *InsertBefore) { if (auto *C = dyn_cast(MemorySemantics)) { return ConstantInt::get(C->getType(), mapSPIRVMemSemanticToOCL(C->getZExtValue()).second); } if (auto *CI = dyn_cast(MemorySemantics)) { Function *F = CI->getCalledFunction(); if (F && F->getName().equals(kSPIRVName::TranslateOCLMemOrder)) { // In case the SPIR-V module was created from an OpenCL program by // *this* SPIR-V generator, we know that the value passed to // __translate_ocl_memory_order is what we should pass to the // OpenCL builtin now. return CI->getArgOperand(0); } } // SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and // therefore, we need to apply mask int Mask = MemorySemanticsMaskNone | MemorySemanticsAcquireMask | MemorySemanticsReleaseMask | MemorySemanticsAcquireReleaseMask | MemorySemanticsSequentiallyConsistentMask; return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemOrder, MemorySemantics, OCLMemOrderMap::getRMap(), /* IsReverse */ true, None, InsertBefore, Mask); } Value *SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags( Value *MemorySemantics, Instruction *InsertBefore) { if (auto *C = dyn_cast(MemorySemantics)) { return ConstantInt::get(C->getType(), mapSPIRVMemSemanticToOCL(C->getZExtValue()).first); } // TODO: any possible optimizations? // SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and // therefore, we need to apply mask int Mask = MemorySemanticsWorkgroupMemoryMask | MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask; return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemFence, MemorySemantics, OCLMemFenceExtendedMap::getRMap(), /* IsReverse */ true, None, InsertBefore, Mask); } void llvm::mangleOpenClBuiltin(const std::string &UniqName, ArrayRef ArgTypes, ArrayRef PointerElementTys, std::string &MangledName) { OCLUtil::OCLBuiltinFuncMangleInfo BtnInfo(ArgTypes); BtnInfo.fillPointerElementTypes(PointerElementTys); MangledName = SPIRV::mangleBuiltin(UniqName, ArgTypes, &BtnInfo); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/OCLUtil.h000066400000000000000000000721031506646577400206130ustar00rootroot00000000000000//===- OCLUtil.h - OCL Utilities declarations -------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file declares OCL utility functions. // //===----------------------------------------------------------------------===// #ifndef SPIRV_OCLUTIL_H #define SPIRV_OCLUTIL_H #include "SPIRVInternal.h" #include "llvm/ADT/SmallString.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/Path.h" #include #include #include #include #include using namespace SPIRV; using namespace llvm; using namespace spv; namespace OCLUtil { /////////////////////////////////////////////////////////////////////////////// // // Enums // /////////////////////////////////////////////////////////////////////////////// enum OCLMemFenceKind { OCLMF_Local = 1, OCLMF_Global = 2, OCLMF_Image = 4, }; // This enum declares extra constants for OpenCL mem_fence flag. It includes // combinations of local/global/image flags. enum OCLMemFenceExtendedKind { OCLMFEx_Local = OCLMF_Local, OCLMFEx_Global = OCLMF_Global, OCLMFEx_Local_Global = OCLMF_Global | OCLMF_Local, OCLMFEx_Image = OCLMF_Image, OCLMFEx_Image_Local = OCLMF_Image | OCLMF_Local, OCLMFEx_Image_Global = OCLMF_Image | OCLMF_Global, OCLMFEx_Image_Local_Global = OCLMF_Image | OCLMF_Global | OCLMF_Local, }; enum OCLScopeKind { OCLMS_work_item, OCLMS_work_group, OCLMS_device, OCLMS_all_svm_devices, OCLMS_sub_group, }; // The enum below declares constants corresponding to memory synchronization // operations constants defined in // https://www.khronos.org/registry/OpenCL/sdk/2.1/docs/man/xhtml/memory_order.html // To avoid any inconsistence here, constants are explicitly initialized with // the corresponding constants from 'std::memory_order' enum. enum OCLMemOrderKind { #if __cplusplus >= 202002L OCLMO_relaxed = static_cast(std::memory_order::relaxed), OCLMO_acquire = static_cast(std::memory_order::acquire), OCLMO_release = static_cast(std::memory_order::release), OCLMO_acq_rel = static_cast(std::memory_order::acq_rel), OCLMO_seq_cst = static_cast(std::memory_order::seq_cst) #else OCLMO_relaxed = std::memory_order::memory_order_relaxed, OCLMO_acquire = std::memory_order::memory_order_acquire, OCLMO_release = std::memory_order::memory_order_release, OCLMO_acq_rel = std::memory_order::memory_order_acq_rel, OCLMO_seq_cst = std::memory_order::memory_order_seq_cst #endif }; enum IntelFPGAMemoryAccessesVal { BurstCoalesce = 0x1, CacheSizeFlag = 0x2, DontStaticallyCoalesce = 0x4, PrefetchFlag = 0x8 }; /////////////////////////////////////////////////////////////////////////////// // // Types // /////////////////////////////////////////////////////////////////////////////// typedef SPIRVMap OCLMemFenceMap; typedef SPIRVMap OCLMemFenceExtendedMap; typedef SPIRVMap OCLMemOrderMap; typedef SPIRVMap OCLMemScopeMap; typedef SPIRVMap SPIRSPIRVGroupOperationMap; typedef SPIRVMap SPIRSPIRVFPRoundingModeMap; typedef SPIRVMap OCLSPIRVBuiltinMap; class OCL12Builtin; typedef SPIRVMap OCL12SPIRVBuiltinMap; typedef SPIRVMap SPIRSPIRVBuiltinVariableMap; /// Tuple of literals for atomic_work_item_fence (flag, order, scope) typedef std::tuple AtomicWorkItemFenceLiterals; /// Tuple of literals for work_group_barrier or sub_group_barrier /// (flag, mem_scope, exec_scope) typedef std::tuple BarrierLiterals; class OCLOpaqueType; typedef SPIRVMap OCLOpaqueTypeOpCodeMap; /// Information for translating OCL builtin. struct OCLBuiltinTransInfo { std::string UniqName; std::string MangledName; std::string Postfix; // Postfix to be added /// Postprocessor of operands std::function &)> PostProc; Type *RetTy; // Return type of the translated function bool IsRetSigned; // When RetTy is int, determines if extensions // on it should be a sext or zet. OCLBuiltinTransInfo() : RetTy(nullptr), IsRetSigned(false) { PostProc = [](std::vector &) {}; } }; /////////////////////////////////////////////////////////////////////////////// // // Constants // /////////////////////////////////////////////////////////////////////////////// namespace kOCLBuiltinName { const static char All[] = "all"; const static char Any[] = "any"; #define _SPIRV_OP(x, y) \ const static char ArbitraryFloat##x##INTEL[] = "intel_arbitrary_float_" #y; _SPIRV_OP(Cast, cast) _SPIRV_OP(CastFromInt, cast_from_int) _SPIRV_OP(CastToInt, cast_to_int) _SPIRV_OP(Add, add) _SPIRV_OP(Sub, sub) _SPIRV_OP(Mul, mul) _SPIRV_OP(Div, div) _SPIRV_OP(GT, gt) _SPIRV_OP(GE, ge) _SPIRV_OP(LT, lt) _SPIRV_OP(LE, le) _SPIRV_OP(EQ, eq) _SPIRV_OP(Recip, recip) _SPIRV_OP(RSqrt, rsqrt) _SPIRV_OP(Cbrt, cbrt) _SPIRV_OP(Hypot, hypot) _SPIRV_OP(Sqrt, sqrt) _SPIRV_OP(Log, log) _SPIRV_OP(Log2, log2) _SPIRV_OP(Log10, log10) _SPIRV_OP(Log1p, log1p) _SPIRV_OP(Exp, exp) _SPIRV_OP(Exp2, exp2) _SPIRV_OP(Exp10, exp10) _SPIRV_OP(Expm1, expm1) _SPIRV_OP(Sin, sin) _SPIRV_OP(Cos, cos) _SPIRV_OP(SinCos, sincos) _SPIRV_OP(SinPi, sinpi) _SPIRV_OP(CosPi, cospi) _SPIRV_OP(SinCosPi, sincospi) _SPIRV_OP(ASin, asin) _SPIRV_OP(ASinPi, asinpi) _SPIRV_OP(ACos, acos) _SPIRV_OP(ACosPi, acospi) _SPIRV_OP(ATan, atan) _SPIRV_OP(ATanPi, atanpi) _SPIRV_OP(ATan2, atan2) _SPIRV_OP(Pow, pow) _SPIRV_OP(PowR, powr) _SPIRV_OP(PowN, pown) #undef _SPIRV_OP const static char AsyncWorkGroupCopy[] = "async_work_group_copy"; const static char AsyncWorkGroupStridedCopy[] = "async_work_group_strided_copy"; const static char AtomPrefix[] = "atom_"; const static char AtomCmpXchg[] = "atom_cmpxchg"; const static char AtomicPrefix[] = "atomic_"; const static char AtomicCmpXchg[] = "atomic_cmpxchg"; const static char AtomicCmpXchgStrong[] = "atomic_compare_exchange_strong"; const static char AtomicCmpXchgStrongExplicit[] = "atomic_compare_exchange_strong_explicit"; const static char AtomicCmpXchgWeak[] = "atomic_compare_exchange_weak"; const static char AtomicCmpXchgWeakExplicit[] = "atomic_compare_exchange_weak_explicit"; const static char AtomicInit[] = "atomic_init"; const static char AtomicWorkItemFence[] = "atomic_work_item_fence"; const static char Barrier[] = "barrier"; const static char Clamp[] = "clamp"; const static char ClockReadPrefix[] = "clock_read_"; const static char ConvertPrefix[] = "convert_"; const static char Dot[] = "dot"; const static char DotAccSat[] = "dot_acc_sat"; const static char Dot4x8PackedPrefix[] = "dot_4x8packed_"; const static char DotAccSat4x8PackedPrefix[] = "dot_acc_sat_4x8packed_"; const static char EnqueueKernel[] = "enqueue_kernel"; const static char FixedSqrtINTEL[] = "intel_arbitrary_fixed_sqrt"; const static char FixedRecipINTEL[] = "intel_arbitrary_fixed_recip"; const static char FixedRsqrtINTEL[] = "intel_arbitrary_fixed_rsqrt"; const static char FixedSinINTEL[] = "intel_arbitrary_fixed_sin"; const static char FixedCosINTEL[] = "intel_arbitrary_fixed_cos"; const static char FixedSinCosINTEL[] = "intel_arbitrary_fixed_sincos"; const static char FixedSinPiINTEL[] = "intel_arbitrary_fixed_sinpi"; const static char FixedCosPiINTEL[] = "intel_arbitrary_fixed_cospi"; const static char FixedSinCosPiINTEL[] = "intel_arbitrary_fixed_sincospi"; const static char FixedLogINTEL[] = "intel_arbitrary_fixed_log"; const static char FixedExpINTEL[] = "intel_arbitrary_fixed_exp"; const static char FMax[] = "fmax"; const static char FMin[] = "fmin"; const static char FPGARegIntel[] = "__builtin_intel_fpga_reg"; const static char GetFence[] = "get_fence"; const static char GetImageArraySize[] = "get_image_array_size"; const static char GetImageChannelOrder[] = "get_image_channel_order"; const static char GetImageChannelDataType[] = "get_image_channel_data_type"; const static char GetImageDepth[] = "get_image_depth"; const static char GetImageDim[] = "get_image_dim"; const static char GetImageHeight[] = "get_image_height"; const static char GetImageWidth[] = "get_image_width"; const static char IsFinite[] = "isfinite"; const static char IsNan[] = "isnan"; const static char IsNormal[] = "isnormal"; const static char IsInf[] = "isinf"; const static char Max[] = "max"; const static char MemFence[] = "mem_fence"; const static char ReadMemFence[] = "read_mem_fence"; const static char WriteMemFence[] = "write_mem_fence"; const static char Min[] = "min"; const static char Mix[] = "mix"; const static char NDRangePrefix[] = "ndrange_"; const static char Pipe[] = "pipe"; const static char ReadImage[] = "read_image"; const static char ReadPipe[] = "read_pipe"; const static char ReadPipeBlockingINTEL[] = "read_pipe_bl"; const static char RoundingPrefix[] = "_r"; const static char Sampled[] = "sampled_"; const static char SampledReadImage[] = "sampled_read_image"; const static char Signbit[] = "signbit"; const static char SmoothStep[] = "smoothstep"; const static char Step[] = "step"; const static char SubGroupPrefix[] = "sub_group_"; const static char SubGroupBarrier[] = "sub_group_barrier"; const static char SubPrefix[] = "sub_"; const static char ToGlobal[] = "to_global"; const static char ToLocal[] = "to_local"; const static char ToPrivate[] = "to_private"; const static char VLoadPrefix[] = "vload"; const static char VLoadAPrefix[] = "vloada"; const static char VLoadHalf[] = "vload_half"; const static char VStorePrefix[] = "vstore"; const static char VStoreAPrefix[] = "vstorea"; const static char WaitGroupEvent[] = "wait_group_events"; const static char WriteImage[] = "write_image"; const static char WorkGroupBarrier[] = "work_group_barrier"; const static char WritePipe[] = "write_pipe"; const static char WritePipeBlockingINTEL[] = "write_pipe_bl"; const static char WorkGroupPrefix[] = "work_group_"; const static char WorkGroupAll[] = "work_group_all"; const static char WorkGroupAny[] = "work_group_any"; const static char SubGroupAll[] = "sub_group_all"; const static char SubGroupAny[] = "sub_group_any"; const static char WorkPrefix[] = "work_"; const static char SubgroupBlockReadINTELPrefix[] = "intel_sub_group_block_read"; const static char SubgroupBlockWriteINTELPrefix[] = "intel_sub_group_block_write"; const static char SubgroupImageMediaBlockINTELPrefix[] = "intel_sub_group_media_block"; const static char SplitBarrierINTELPrefix[] = "intel_work_group_barrier_"; const static char LDEXP[] = "ldexp"; #define _SPIRV_OP(x) \ const static char ConvertBFloat16##x##AsUShort##x[] = \ "intel_convert_bfloat16" #x "_as_ushort" #x; _SPIRV_OP() _SPIRV_OP(2) _SPIRV_OP(3) _SPIRV_OP(4) _SPIRV_OP(8) _SPIRV_OP(16) #undef _SPIRV_OP #define _SPIRV_OP(x) \ const static char ConvertAsBFloat16##x##Float##x[] = \ "intel_convert_as_bfloat16" #x "_float" #x; _SPIRV_OP() _SPIRV_OP(2) _SPIRV_OP(3) _SPIRV_OP(4) _SPIRV_OP(8) _SPIRV_OP(16) #undef _SPIRV_OP } // namespace kOCLBuiltinName /// Offset for OpenCL image channel order enumeration values. const unsigned int OCLImageChannelOrderOffset = 0x10B0; /// Offset for OpenCL image channel data type enumeration values. const unsigned int OCLImageChannelDataTypeOffset = 0x10D0; /// OCL 1.x atomic memory order when translated to 2.0 atomics. const OCLMemOrderKind OCLLegacyAtomicMemOrder = OCLMO_relaxed; /// OCL 1.x atomic memory scope when translated to 2.0 atomics. const OCLScopeKind OCLLegacyAtomicMemScope = OCLMS_work_group; namespace kOCLVer { const unsigned CL12 = 102000; const unsigned CL20 = 200000; const unsigned CL21 = 201000; const unsigned CL30 = 300000; } // namespace kOCLVer namespace OclExt { // clang-format off enum Kind { #define _SPIRV_OP(x) x, _SPIRV_OP(cl_images) _SPIRV_OP(cl_doubles) _SPIRV_OP(cl_khr_int64_base_atomics) _SPIRV_OP(cl_khr_int64_extended_atomics) _SPIRV_OP(cl_khr_fp16) _SPIRV_OP(cl_khr_gl_sharing) _SPIRV_OP(cl_khr_gl_event) _SPIRV_OP(cl_khr_d3d10_sharing) _SPIRV_OP(cl_khr_media_sharing) _SPIRV_OP(cl_khr_d3d11_sharing) _SPIRV_OP(cl_khr_global_int32_base_atomics) _SPIRV_OP(cl_khr_global_int32_extended_atomics) _SPIRV_OP(cl_khr_local_int32_base_atomics) _SPIRV_OP(cl_khr_local_int32_extended_atomics) _SPIRV_OP(cl_khr_byte_addressable_store) _SPIRV_OP(cl_khr_3d_image_writes) _SPIRV_OP(cl_khr_gl_msaa_sharing) _SPIRV_OP(cl_khr_depth_images) _SPIRV_OP(cl_khr_gl_depth_images) _SPIRV_OP(cl_khr_subgroups) _SPIRV_OP(cl_khr_mipmap_image) _SPIRV_OP(cl_khr_mipmap_image_writes) _SPIRV_OP(cl_khr_egl_event) _SPIRV_OP(cl_khr_srgb_image_writes) _SPIRV_OP(cl_khr_extended_bit_ops) #undef _SPIRV_OP }; // clang-format on } // namespace OclExt namespace kOCLSubgroupsAVCIntel { const static char Prefix[] = "intel_sub_group_avc_"; const static char MCEPrefix[] = "intel_sub_group_avc_mce_"; const static char IMEPrefix[] = "intel_sub_group_avc_ime_"; const static char REFPrefix[] = "intel_sub_group_avc_ref_"; const static char SICPrefix[] = "intel_sub_group_avc_sic_"; const static char TypePrefix[] = "opencl.intel_sub_group_avc_"; } // namespace kOCLSubgroupsAVCIntel /////////////////////////////////////////////////////////////////////////////// // // Functions // /////////////////////////////////////////////////////////////////////////////// /// Get instruction index for SPIR-V extended instruction for OpenCL.std /// extended instruction set. /// \param MangledName The mangled name of OpenCL builtin function. /// \param DemangledName The demangled name of OpenCL builtin function if /// not empty. /// \return instruction index of extended instruction if the OpenCL builtin /// function is translated to an extended instruction, otherwise ~0U. unsigned getExtOp(StringRef MangledName, StringRef DemangledName = ""); /// Get literal arguments of call of atomic_work_item_fence. AtomicWorkItemFenceLiterals getAtomicWorkItemFenceLiterals(CallInst *CI); /// Get literal arguments of call of work_group_barrier or sub_group_barrier. BarrierLiterals getBarrierLiterals(CallInst *CI); /// Get number of memory order arguments for atomic builtin function. size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name); /// Get number of memory order arguments for spirv atomic builtin function. size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC); /// Return true for OpenCL builtins which do compute operations /// (like add, sub, min, max, inc, dec, ...) atomically bool isComputeAtomicOCLBuiltin(StringRef DemangledName); /// Get OCL version from metadata opencl.ocl.version. /// \param AllowMulti Allows multiple operands if true. /// \return OCL version encoded as Major*10^5+Minor*10^3+Rev, /// e.g. 201000 for OCL 2.1, 200000 for OCL 2.0, 102000 for OCL 1.2, /// 0 if metadata not found. /// If there are multiple operands, check they are identical. unsigned getOCLVersion(Module *M, bool AllowMulti = false); /// Encode OpenCL version as Major*10^5+Minor*10^3+Rev. unsigned encodeOCLVer(unsigned short Major, unsigned char Minor, unsigned char Rev); /// Decode OpenCL version which is encoded as Major*10^5+Minor*10^3+Rev std::tuple decodeOCLVer(unsigned Ver); /// Decode a MDNode assuming it contains three integer constants. void decodeMDNode(MDNode *N, unsigned &X, unsigned &Y, unsigned &Z); /// Get full path from debug info metadata /// Return empty string if the path is not available. template std::string getFullPath(const T *Scope) { if (!Scope) return std::string(); StringRef Filename = Scope->getFilename(); auto Style = sys::path::Style::native; if (sys::path::is_absolute(Filename, Style)) return Filename.str(); SmallString<16> DirName = Scope->getDirectory(); sys::path::append(DirName, Style, Filename.str()); return DirName.str().str(); } /// Decode OpenCL vector type hint MDNode and encode it as SPIR-V execution /// mode VecTypeHint. unsigned transVecTypeHint(MDNode *Node); /// Decode SPIR-V encoding of vector type hint execution mode. Type *decodeVecTypeHint(LLVMContext &C, unsigned Code); SPIRAddressSpace getOCLOpaqueTypeAddrSpace(Op OpCode); SPIR::TypeAttributeEnum getOCLOpaqueTypeAddrSpace(SPIR::TypePrimitiveEnum Prim); inline unsigned mapOCLMemSemanticToSPIRV(unsigned MemFenceFlag, OCLMemOrderKind Order) { return OCLMemOrderMap::map(Order) | mapBitMask(MemFenceFlag); } inline unsigned mapOCLMemFenceFlagToSPIRV(unsigned MemFenceFlag) { return mapBitMask(MemFenceFlag); } inline std::pair mapSPIRVMemSemanticToOCL(unsigned Sema) { return std::make_pair( rmapBitMask(Sema), OCLMemOrderMap::rmap(extractSPIRVMemOrderSemantic(Sema))); } inline OCLMemOrderKind mapSPIRVMemOrderToOCL(unsigned Sema) { return OCLMemOrderMap::rmap(extractSPIRVMemOrderSemantic(Sema)); } /// Mutate call instruction to call OpenCL builtin function. CallInst *mutateCallInstOCL( Module *M, CallInst *CI, std::function &)> ArgMutate, AttributeList *Attrs = nullptr); /// Mutate call instruction to call OpenCL builtin function. Instruction *mutateCallInstOCL( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, AttributeList *Attrs = nullptr, bool TakeFuncName = false); /// If the value is a special type initializer (something that bitcasts from /// spirv.ConstantSampler to spirv.Sampler or likewise for PipeStorage), get the /// original type initializer, unwrap the bitcast. Otherwise, return nullptr. Value *unwrapSpecialTypeInitializer(Value *V); bool isPipeOrAddressSpaceCastBI(const StringRef MangledName); bool isEnqueueKernelBI(const StringRef MangledName); bool isKernelQueryBI(const StringRef MangledName); /// Check that the type is the sampler_t bool isSamplerStructTy(StructType *Ty); // Checks if the binary operator is an unfused fmul + fadd instruction. bool isUnfusedMulAdd(BinaryOperator *B); // Get data and vector size postfix for sugroup_block_{read|write} builtins // as specified by cl_intel_subgroups* extensions. // Scalar data assumed to be represented as vector of one element. std::string getIntelSubgroupBlockDataPostfix(unsigned ElementBitSize, unsigned VectorNumElements); void insertImageNameAccessQualifier(SPIRVAccessQualifierKind Acc, std::string &Name); } // namespace OCLUtil using namespace OCLUtil; namespace SPIRV { template Instruction * getOrCreateSwitchFunc(StringRef MapName, Value *V, const SPIRVMap &Map, bool IsReverse, Optional DefaultCase, Instruction *InsertPoint, int KeyMask = 0) { static_assert(std::is_convertible::value && std::is_convertible::value, "Can map only integer values"); Type *Ty = V->getType(); assert(Ty && Ty->isIntegerTy() && "Can't map non-integer types"); Module *M = InsertPoint->getModule(); Function *F = getOrCreateFunction(M, Ty, Ty, MapName); if (!F->empty()) // The switch function already exists. just call it. return addCallInst(M, MapName, Ty, V, nullptr, InsertPoint); F->setLinkage(GlobalValue::PrivateLinkage); LLVMContext &Ctx = M->getContext(); BasicBlock *BB = BasicBlock::Create(Ctx, "entry", F); IRBuilder<> IRB(BB); SwitchInst *SI; F->arg_begin()->setName("key"); if (KeyMask) { Value *MaskV = ConstantInt::get(Type::getInt32Ty(Ctx), KeyMask); Value *NewKey = IRB.CreateAnd(MaskV, F->arg_begin()); NewKey->setName("key.masked"); SI = IRB.CreateSwitch(NewKey, BB); } else { SI = IRB.CreateSwitch(F->arg_begin(), BB); } if (!DefaultCase) { BasicBlock *DefaultBB = BasicBlock::Create(Ctx, "default", F); IRBuilder<> DefaultIRB(DefaultBB); DefaultIRB.CreateUnreachable(); SI->setDefaultDest(DefaultBB); } Map.foreach ([&](int Key, int Val) { if (IsReverse) std::swap(Key, Val); BasicBlock *CaseBB = BasicBlock::Create(Ctx, "case." + Twine(Key), F); IRBuilder<> CaseIRB(CaseBB); CaseIRB.CreateRet(CaseIRB.getInt32(Val)); SI->addCase(IRB.getInt32(Key), CaseBB); if (Key == DefaultCase) SI->setDefaultDest(CaseBB); }); assert(SI->getDefaultDest() != BB && "Invalid default destination in switch"); return addCallInst(M, MapName, Ty, V, nullptr, InsertPoint); } /// Performs conversion from OpenCL memory_scope into SPIR-V Scope. /// /// Supports both constant and non-constant values. To handle the latter case, /// function with switch..case statement will be inserted into module which /// \arg InsertBefore belongs to (in order to perform mapping at runtime) /// /// \param [in] MemScope memory_scope value which needs to be translated /// \param [in] DefaultCase default value for switch..case construct if /// dynamic mapping is used /// \param [in] InsertBefore insertion point for call into conversion function /// which is generated if \arg MemScope is not a constant /// \returns \c Value corresponding to SPIR-V Scope equivalent to OpenCL /// memory_scope passed in \arg MemScope Value *transOCLMemScopeIntoSPIRVScope(Value *MemScope, Optional DefaultCase, Instruction *InsertBefore); /// Performs conversion from OpenCL memory_order into SPIR-V Memory Semantics. /// /// Supports both constant and non-constant values. To handle the latter case, /// function with switch..case statement will be inserted into module which /// \arg InsertBefore belongs to (in order to perform mapping at runtime) /// /// \param [in] MemOrder memory_scope value which needs to be translated /// \param [in] DefaultCase default value for switch..case construct if /// dynamic mapping is used /// \param [in] InsertBefore insertion point for call into conversion function /// which is generated if \arg MemOrder is not a constant /// \returns \c Value corresponding to SPIR-V Memory Semantics equivalent to /// OpenCL memory_order passed in \arg MemOrder Value *transOCLMemOrderIntoSPIRVMemorySemantics(Value *MemOrder, Optional DefaultCase, Instruction *InsertBefore); /// Performs conversion from SPIR-V Scope into OpenCL memory_scope. /// /// Supports both constant and non-constant values. To handle the latter case, /// function with switch..case statement will be inserted into module which /// \arg InsertBefore belongs to (in order to perform mapping at runtime) /// /// \param [in] MemScope Scope value which needs to be translated /// \param [in] InsertBefore insertion point for call into conversion function /// which is generated if \arg MemScope is not a constant /// \returns \c Value corresponding to OpenCL memory_scope equivalent to SPIR-V /// Scope passed in \arg MemScope Value *transSPIRVMemoryScopeIntoOCLMemoryScope(Value *MemScope, Instruction *InsertBefore); /// Performs conversion from SPIR-V Memory Semantics into OpenCL memory_order. /// /// Supports both constant and non-constant values. To handle the latter case, /// function with switch..case statement will be inserted into module which /// \arg InsertBefore belongs to (in order to perform mapping at runtime) /// /// \param [in] MemorySemantics Memory Semantics value which needs to be /// translated /// \param [in] InsertBefore insertion point for call into conversion function /// which is generated if \arg MemorySemantics is not a constant /// \returns \c Value corresponding to OpenCL memory_order equivalent to SPIR-V /// Memory Semantics passed in \arg MemorySemantics Value *transSPIRVMemorySemanticsIntoOCLMemoryOrder(Value *MemorySemantics, Instruction *InsertBefore); /// Performs conversion from SPIR-V Memory Semantics into OpenCL /// mem_fence_flags. /// /// Supports both constant and non-constant values. To handle the latter case, /// function with switch..case statement will be inserted into module which /// \arg InsertBefore belongs to (in order to perform mapping at runtime) /// /// \param [in] MemorySemantics Memory Semantics value which needs to be /// translated /// \param [in] InsertBefore insertion point for call into conversion function /// which is generated if \arg MemorySemantics is not a constant /// \returns \c Value corresponding to OpenCL mem_fence_flags equivalent to /// SPIR-V Memory Semantics passed in \arg MemorySemantics Value *transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Value *MemorySemantics, Instruction *InsertBefore); class SPIRVSubgroupsAVCIntelInst; typedef SPIRVMap OCLSPIRVSubgroupAVCIntelBuiltinMap; typedef SPIRVMap LLVMSPIRVAtomicRmwOpCodeMap; class SPIRVFixedPointIntelInst; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add("intel_arbitrary_fixed_" #x, OpFixed##y##INTEL); _SPIRV_OP(sqrt, Sqrt) _SPIRV_OP(recip, Recip) _SPIRV_OP(rsqrt, Rsqrt) _SPIRV_OP(sin, Sin) _SPIRV_OP(cos, Cos) _SPIRV_OP(sincos, SinCos) _SPIRV_OP(sinpi, SinPi) _SPIRV_OP(cospi, CosPi) _SPIRV_OP(sincospi, SinCosPi) _SPIRV_OP(log, Log) _SPIRV_OP(exp, Exp) #undef _SPIRV_OP } typedef SPIRVMap SPIRVFixedPointIntelMap; class SPIRVArbFloatIntelInst; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) \ add("intel_arbitrary_float_" #y, OpArbitraryFloat##x##INTEL); _SPIRV_OP(Cast, cast) _SPIRV_OP(CastFromInt, cast_from_int) _SPIRV_OP(CastToInt, cast_to_int) _SPIRV_OP(Add, add) _SPIRV_OP(Sub, sub) _SPIRV_OP(Mul, mul) _SPIRV_OP(Div, div) _SPIRV_OP(GT, gt) _SPIRV_OP(GE, ge) _SPIRV_OP(LT, lt) _SPIRV_OP(LE, le) _SPIRV_OP(EQ, eq) _SPIRV_OP(Recip, recip) _SPIRV_OP(RSqrt, rsqrt) _SPIRV_OP(Cbrt, cbrt) _SPIRV_OP(Hypot, hypot) _SPIRV_OP(Sqrt, sqrt) _SPIRV_OP(Log, log) _SPIRV_OP(Log2, log2) _SPIRV_OP(Log10, log10) _SPIRV_OP(Log1p, log1p) _SPIRV_OP(Exp, exp) _SPIRV_OP(Exp2, exp2) _SPIRV_OP(Exp10, exp10) _SPIRV_OP(Expm1, expm1) _SPIRV_OP(Sin, sin) _SPIRV_OP(Cos, cos) _SPIRV_OP(SinCos, sincos) _SPIRV_OP(SinPi, sinpi) _SPIRV_OP(CosPi, cospi) _SPIRV_OP(SinCosPi, sincospi) _SPIRV_OP(ASin, asin) _SPIRV_OP(ASinPi, asinpi) _SPIRV_OP(ACos, acos) _SPIRV_OP(ACosPi, acospi) _SPIRV_OP(ATan, atan) _SPIRV_OP(ATanPi, atanpi) _SPIRV_OP(ATan2, atan2) _SPIRV_OP(Pow, pow) _SPIRV_OP(PowR, powr) _SPIRV_OP(PowN, pown) #undef _SPIRV_OP } typedef SPIRVMap SPIRVArbFloatIntelMap; } // namespace SPIRV #endif // SPIRV_OCLUTIL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/PreprocessMetadata.cpp000066400000000000000000000314611506646577400234630ustar00rootroot00000000000000//===- PreprocessMetadata.cpp - - C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements preprocessing of LLVM IR metadata in order to perform // further translation to SPIR-V. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "clmdtospv" #include "PreprocessMetadata.h" #include "OCLUtil.h" #include "SPIRVInternal.h" #include "SPIRVMDBuilder.h" #include "SPIRVMDWalker.h" #include "VectorComputeUtil.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/Support/CommandLine.h" using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { cl::opt EraseOCLMD("spirv-erase-cl-md", cl::init(true), cl::desc("Erase OpenCL metadata")); char PreprocessMetadataLegacy::ID = 0; bool PreprocessMetadataLegacy::runOnModule(Module &Module) { return runPreprocessMetadata(Module); } llvm::PreservedAnalyses PreprocessMetadataPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runPreprocessMetadata(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } bool PreprocessMetadataBase::runPreprocessMetadata(Module &Module) { M = &Module; Ctx = &M->getContext(); LLVM_DEBUG(dbgs() << "Enter PreprocessMetadata:\n"); visit(M); LLVM_DEBUG(dbgs() << "After PreprocessMetadata:\n" << *M); verifyRegularizationPass(*M, "PreprocessMetadata"); return true; } void PreprocessMetadataBase::preprocessCXXStructorList( SPIRVMDBuilder::NamedMDWrapper &EM, GlobalVariable *V, ExecutionMode EMode) { auto *List = dyn_cast_or_null(V->getInitializer()); if (!List) return; for (Value *V : List->operands()) { auto *Structor = cast(V); // Each entry in the list is a struct containing 3 members: // (priority, function, data), with function being the entry point. auto *Kernel = cast(Structor->getOperand(1)); EM.addOp().add(Kernel).add(EMode).done(); } } void PreprocessMetadataBase::visit(Module *M) { SPIRVMDBuilder B(*M); SPIRVMDWalker W(*M); preprocessOCLMetadata(M, &B, &W); preprocessVectorComputeMetadata(M, &B, &W); // Create metadata representing (empty so far) list // of OpExecutionMode instructions auto EM = B.addNamedMD(kSPIRVMD::ExecutionMode); // !spirv.ExecutionMode = {} // Process special variables in LLVM IR module. if (auto *GV = M->getGlobalVariable("llvm.global_ctors")) preprocessCXXStructorList(EM, GV, spv::ExecutionModeInitializer); // Add execution modes for kernels. We take it from metadata attached to // the kernel functions. for (Function &Kernel : *M) { if (Kernel.getCallingConv() != CallingConv::SPIR_KERNEL) continue; // Specifing execution modes for the Kernel and adding it to the list // of ExecutionMode instructions. // !{void (i32 addrspace(1)*)* @kernel, i32 17, i32 X, i32 Y, i32 Z} if (MDNode *WGSize = Kernel.getMetadata(kSPIR2MD::WGSize)) { unsigned X, Y, Z; decodeMDNode(WGSize, X, Y, Z); EM.addOp() .add(&Kernel) .add(spv::ExecutionModeLocalSize) .add(X) .add(Y) .add(Z) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 18, i32 X, i32 Y, i32 Z} if (MDNode *WGSizeHint = Kernel.getMetadata(kSPIR2MD::WGSizeHint)) { unsigned X, Y, Z; decodeMDNode(WGSizeHint, X, Y, Z); EM.addOp() .add(&Kernel) .add(spv::ExecutionModeLocalSizeHint) .add(X) .add(Y) .add(Z) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 30, i32 hint} if (MDNode *VecTypeHint = Kernel.getMetadata(kSPIR2MD::VecTyHint)) { EM.addOp() .add(&Kernel) .add(spv::ExecutionModeVecTypeHint) .add(transVecTypeHint(VecTypeHint)) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 35, i32 size} if (MDNode *ReqdSubgroupSize = Kernel.getMetadata(kSPIR2MD::SubgroupSize)) { EM.addOp() .add(&Kernel) .add(spv::ExecutionModeSubgroupSize) .add(getMDOperandAsInt(ReqdSubgroupSize, 0)) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 max_work_group_size, i32 X, // i32 Y, i32 Z} if (MDNode *MaxWorkgroupSizeINTEL = Kernel.getMetadata(kSPIR2MD::MaxWGSize)) { unsigned X, Y, Z; decodeMDNode(MaxWorkgroupSizeINTEL, X, Y, Z); EM.addOp() .add(&Kernel) .add(spv::ExecutionModeMaxWorkgroupSizeINTEL) .add(X) .add(Y) .add(Z) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 no_global_work_offset} if (Kernel.getMetadata(kSPIR2MD::NoGlobalOffset)) { EM.addOp().add(&Kernel).add(spv::ExecutionModeNoGlobalOffsetINTEL).done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 max_global_work_dim, i32 dim} if (MDNode *MaxWorkDimINTEL = Kernel.getMetadata(kSPIR2MD::MaxWGDim)) { EM.addOp() .add(&Kernel) .add(spv::ExecutionModeMaxWorkDimINTEL) .add(getMDOperandAsInt(MaxWorkDimINTEL, 0)) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 num_simd_work_items, i32 num} if (MDNode *NumSIMDWorkitemsINTEL = Kernel.getMetadata(kSPIR2MD::NumSIMD)) { EM.addOp() .add(&Kernel) .add(spv::ExecutionModeNumSIMDWorkitemsINTEL) .add(getMDOperandAsInt(NumSIMDWorkitemsINTEL, 0)) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 scheduler_target_fmax_mhz, // i32 num} if (MDNode *SchedulerTargetFmaxMhzINTEL = Kernel.getMetadata(kSPIR2MD::FmaxMhz)) { EM.addOp() .add(&Kernel) .add(spv::ExecutionModeSchedulerTargetFmaxMhzINTEL) .add(getMDOperandAsInt(SchedulerTargetFmaxMhzINTEL, 0)) .done(); } // !{void (i32 addrspace(1)*)* @kernel, i32 ip_interface, i32 interface} if (MDNode *Interface = Kernel.getMetadata(kSPIR2MD::IntelFPGAIPInterface)) { std::set InterfaceStrSet; // Default mode is 'csr' aka !ip_interface !N // !N = !{!”csr”} // don't emit any particular SPIR-V for it // Streaming mode metadata be like: // Not 'stall free' mode (to be mapped on '0' literal) // !ip_interface !N // !N = !{!"streaming"} // 'stall free' mode (to be mapped on '1' literal) // !ip_interface !N // !N = !{!"streaming", !"stall_free_return"} for (size_t I = 0; I != Interface->getNumOperands(); ++I) InterfaceStrSet.insert(getMDOperandAsString(Interface, I).str()); if (InterfaceStrSet.find("streaming") != InterfaceStrSet.end()) { int32_t InterfaceMode = 0; if (InterfaceStrSet.find("stall_free_return") != InterfaceStrSet.end()) InterfaceMode = 1; EM.addOp() .add(&Kernel) .add(spv::ExecutionModeStreamingInterfaceINTEL) .add(InterfaceMode) .done(); } } } } void PreprocessMetadataBase::preprocessOCLMetadata(Module *M, SPIRVMDBuilder *B, SPIRVMDWalker *W) { unsigned CLVer = getOCLVersion(M, true); if (CLVer == 0) return; // Preprocess OpenCL-specific metadata // !spirv.Source = !{!x} // !{x} = !{i32 3, i32 102000} B->addNamedMD(kSPIRVMD::Source) .addOp() .add(CLVer == kOCLVer::CL21 ? spv::SourceLanguageOpenCL_CPP : spv::SourceLanguageOpenCL_C) .add(CLVer) .done(); if (EraseOCLMD) B->eraseNamedMD(kSPIR2MD::OCLVer).eraseNamedMD(kSPIR2MD::SPIRVer); // !spirv.MemoryModel = !{!x} // !{x} = !{i32 1, i32 2} Triple TT(M->getTargetTriple()); assert(isSupportedTriple(TT) && "Invalid triple"); B->addNamedMD(kSPIRVMD::MemoryModel) .addOp() .add(TT.isArch32Bit() ? spv::AddressingModelPhysical32 : spv::AddressingModelPhysical64) .add(spv::MemoryModelOpenCL) .done(); // Add source extensions // !spirv.SourceExtension = !{!x, !y, ...} // !x = {!"cl_khr_..."} // !y = {!"cl_khr_..."} auto Exts = getNamedMDAsStringSet(M, kSPIR2MD::Extensions); if (!Exts.empty()) { auto N = B->addNamedMD(kSPIRVMD::SourceExtension); for (auto &I : Exts) N.addOp().add(I).done(); } if (EraseOCLMD) B->eraseNamedMD(kSPIR2MD::Extensions).eraseNamedMD(kSPIR2MD::OptFeatures); if (EraseOCLMD) B->eraseNamedMD(kSPIR2MD::FPContract); } void PreprocessMetadataBase::preprocessVectorComputeMetadata(Module *M, SPIRVMDBuilder *B, SPIRVMDWalker *W) { using namespace VectorComputeUtil; auto EM = B->addNamedMD(kSPIRVMD::ExecutionMode); for (auto &F : *M) { if (F.getCallingConv() != CallingConv::SPIR_KERNEL) continue; // Add VC float control execution modes // RoundMode and FloatMode are always same for all types in VC // While Denorm could be different for double, float and half auto Attrs = F.getAttributes(); if (Attrs.hasFnAttr(kVCMetadata::VCFloatControl)) { SPIRVWord Mode = 0; Attrs.getFnAttr(kVCMetadata::VCFloatControl) .getValueAsString() .getAsInteger(0, Mode); spv::ExecutionMode ExecRoundMode = FPRoundingModeExecModeMap::map(getFPRoundingMode(Mode)); spv::ExecutionMode ExecFloatMode = FPOperationModeExecModeMap::map(getFPOperationMode(Mode)); VCFloatTypeSizeMap::foreach ( [&](VCFloatType FloatType, unsigned TargetWidth) { EM.addOp().add(&F).add(ExecRoundMode).add(TargetWidth).done(); EM.addOp().add(&F).add(ExecFloatMode).add(TargetWidth).done(); EM.addOp() .add(&F) .add(FPDenormModeExecModeMap::map( getFPDenormMode(Mode, FloatType))) .add(TargetWidth) .done(); }); } if (Attrs.hasFnAttr(kVCMetadata::VCSLMSize)) { SPIRVWord SLMSize = 0; Attrs.getFnAttr(kVCMetadata::VCSLMSize) .getValueAsString() .getAsInteger(0, SLMSize); EM.addOp() .add(&F) .add(spv::ExecutionModeSharedLocalMemorySizeINTEL) .add(SLMSize) .done(); } if (Attrs.hasFnAttr(kVCMetadata::VCNamedBarrierCount)) { SPIRVWord NBarrierCnt = 0; Attrs.getFnAttr(kVCMetadata::VCNamedBarrierCount) .getValueAsString() .getAsInteger(0, NBarrierCnt); EM.addOp() .add(&F) .add(spv::ExecutionModeNamedBarrierCountINTEL) .add(NBarrierCnt) .done(); } } } } // namespace SPIRV INITIALIZE_PASS(PreprocessMetadataLegacy, "preprocess-metadata", "Transform LLVM IR metadata to SPIR-V metadata format", false, false) ModulePass *llvm::createPreprocessMetadataLegacy() { return new PreprocessMetadataLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/PreprocessMetadata.h000066400000000000000000000065641506646577400231360ustar00rootroot00000000000000//=- PreprocessMetadata.h - Metadata preprocessing pass -*- C++ -*-=// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements preprocessing of LLVM IR metadata in order to perform // further translation to SPIR-V. // //===----------------------------------------------------------------------===// #ifndef SPIRV_PREPROCESSMETADATA_H #define SPIRV_PREPROCESSMETADATA_H #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "SPIRVMDBuilder.h" namespace SPIRV { class SPIRVMDWalker; class PreprocessMetadataBase { public: PreprocessMetadataBase() : M(nullptr), Ctx(nullptr) {} bool runPreprocessMetadata(Module &M); void visit(Module *M); void preprocessCXXStructorList(SPIRVMDBuilder::NamedMDWrapper &EM, GlobalVariable *V, ExecutionMode EMode); void preprocessOCLMetadata(Module *M, SPIRVMDBuilder *B, SPIRVMDWalker *W); void preprocessVectorComputeMetadata(Module *M, SPIRVMDBuilder *B, SPIRVMDWalker *W); private: Module *M; LLVMContext *Ctx; }; class PreprocessMetadataLegacy : public ModulePass, public PreprocessMetadataBase { public: PreprocessMetadataLegacy() : ModulePass(ID) { initializePreprocessMetadataLegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override; static char ID; }; class PreprocessMetadataPass : public llvm::PassInfoMixin, public PreprocessMetadataBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; } // namespace SPIRV #endif // SPIRV_PREPROCESSMETADATA_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVInternal.h000066400000000000000000001277731506646577400217560ustar00rootroot00000000000000//===- LLVMSPIRVInternal.h - SPIR-V internal header file -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file declares classes and functions shared by SPIR-V reader/writer. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVINTERNAL_H #define SPIRV_SPIRVINTERNAL_H #include "NameMangleAPI.h" #include "libSPIRV/SPIRVEnum.h" #include "libSPIRV/SPIRVError.h" #include "libSPIRV/SPIRVNameMapEnum.h" #include "libSPIRV/SPIRVType.h" #include "libSPIRV/SPIRVUtil.h" #include "LLVMSPIRVLib.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include #include using namespace SPIRV; using namespace llvm; namespace llvm { class IntrinsicInst; class IRBuilderBase; } namespace SPIRV { /// The LLVM/SPIR-V translator version used to fill the lower 16 bits of the /// generator's magic number in the generated SPIR-V module. /// This number should be bumped up whenever the generated SPIR-V changes. const static unsigned short KTranslatorVer = 14; #define SPCV_TARGET_LLVM_IMAGE_TYPE_ENCODE_ACCESS_QUAL 0 // Workaround for SPIR 2 producer bug about kernel function calling convention. // This workaround checks metadata to determine if a function is kernel. #define SPCV_RELAX_KERNEL_CALLING_CONV 1 class SPIRVOpaqueType; typedef SPIRVMap SPIRVOpaqueTypeOpCodeMap; // Ad hoc function used by LLVM/SPIRV converter for type casting #define SPCV_CAST "spcv.cast" #define LLVM_MEMCPY "llvm.memcpy" // The name of function generated by Clang to initialize sampler(which is // opaqueue type) by 32-bit integer. The name is taken from // CodeGenModule::createOpenCLIntToSamplerConversion(). #define SAMPLER_INIT "__translate_sampler_initializer" template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(Instruction::x, Op##y); /* Casts */ _SPIRV_OP(ZExt, UConvert) _SPIRV_OP(SExt, SConvert) _SPIRV_OP(Trunc, UConvert) _SPIRV_OP(FPToUI, ConvertFToU) _SPIRV_OP(FPToSI, ConvertFToS) _SPIRV_OP(UIToFP, ConvertUToF) _SPIRV_OP(SIToFP, ConvertSToF) _SPIRV_OP(FPTrunc, FConvert) _SPIRV_OP(FPExt, FConvert) _SPIRV_OP(PtrToInt, ConvertPtrToU) _SPIRV_OP(IntToPtr, ConvertUToPtr) _SPIRV_OP(BitCast, Bitcast) _SPIRV_OP(AddrSpaceCast, GenericCastToPtr) _SPIRV_OP(GetElementPtr, AccessChain) _SPIRV_OP(FNeg, FNegate) /*Binary*/ _SPIRV_OP(And, BitwiseAnd) _SPIRV_OP(Or, BitwiseOr) _SPIRV_OP(Xor, BitwiseXor) _SPIRV_OP(Add, IAdd) _SPIRV_OP(FAdd, FAdd) _SPIRV_OP(Sub, ISub) _SPIRV_OP(FSub, FSub) _SPIRV_OP(Mul, IMul) _SPIRV_OP(FMul, FMul) _SPIRV_OP(UDiv, UDiv) _SPIRV_OP(SDiv, SDiv) _SPIRV_OP(FDiv, FDiv) _SPIRV_OP(SRem, SRem) _SPIRV_OP(FRem, FRem) _SPIRV_OP(URem, UMod) _SPIRV_OP(Shl, ShiftLeftLogical) _SPIRV_OP(LShr, ShiftRightLogical) _SPIRV_OP(AShr, ShiftRightArithmetic) #undef _SPIRV_OP } typedef SPIRVMap OpCodeMap; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(CmpInst::x, Op##y); _SPIRV_OP(FCMP_OEQ, FOrdEqual) _SPIRV_OP(FCMP_OGT, FOrdGreaterThan) _SPIRV_OP(FCMP_OGE, FOrdGreaterThanEqual) _SPIRV_OP(FCMP_OLT, FOrdLessThan) _SPIRV_OP(FCMP_OLE, FOrdLessThanEqual) _SPIRV_OP(FCMP_ONE, FOrdNotEqual) _SPIRV_OP(FCMP_ORD, Ordered) _SPIRV_OP(FCMP_UNO, Unordered) _SPIRV_OP(FCMP_UEQ, FUnordEqual) _SPIRV_OP(FCMP_UGT, FUnordGreaterThan) _SPIRV_OP(FCMP_UGE, FUnordGreaterThanEqual) _SPIRV_OP(FCMP_ULT, FUnordLessThan) _SPIRV_OP(FCMP_ULE, FUnordLessThanEqual) _SPIRV_OP(FCMP_UNE, FUnordNotEqual) _SPIRV_OP(ICMP_EQ, IEqual) _SPIRV_OP(ICMP_NE, INotEqual) _SPIRV_OP(ICMP_UGT, UGreaterThan) _SPIRV_OP(ICMP_UGE, UGreaterThanEqual) _SPIRV_OP(ICMP_ULT, ULessThan) _SPIRV_OP(ICMP_ULE, ULessThanEqual) _SPIRV_OP(ICMP_SGT, SGreaterThan) _SPIRV_OP(ICMP_SGE, SGreaterThanEqual) _SPIRV_OP(ICMP_SLT, SLessThan) _SPIRV_OP(ICMP_SLE, SLessThanEqual) #undef _SPIRV_OP } typedef SPIRVMap CmpMap; class IntBoolOpMapId; template <> inline void SPIRVMap::init() { add(OpNot, OpLogicalNot); add(OpBitwiseAnd, OpLogicalAnd); add(OpBitwiseOr, OpLogicalOr); add(OpBitwiseXor, OpLogicalNotEqual); add(OpIEqual, OpLogicalEqual); add(OpINotEqual, OpLogicalNotEqual); } typedef SPIRVMap IntBoolOpMap; #define SPIR_TARGETTRIPLE32 "spir-unknown-unknown" #define SPIR_TARGETTRIPLE64 "spir64-unknown-unknown" #define SPIR_DATALAYOUT32 \ "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" #define SPIR_DATALAYOUT64 \ "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32" \ "-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32" \ "-v32:32:32-v48:64:64-v64:64:64-v96:128:128" \ "-v128:128:128-v192:256:256-v256:256:256" \ "-v512:512:512-v1024:1024:1024" enum SPIRAddressSpace { SPIRAS_Private, SPIRAS_Global, SPIRAS_Constant, SPIRAS_Local, SPIRAS_Generic, SPIRAS_GlobalDevice, SPIRAS_GlobalHost, SPIRAS_Input, SPIRAS_Output, SPIRAS_CodeSectionINTEL, SPIRAS_Count, }; template <> inline void SPIRVMap::init() { add(SPIRAS_Private, "Private"); add(SPIRAS_Global, "Global"); add(SPIRAS_Constant, "Constant"); add(SPIRAS_Local, "Local"); add(SPIRAS_Generic, "Generic"); add(SPIRAS_Input, "Input"); add(SPIRAS_CodeSectionINTEL, "CodeSectionINTEL"); add(SPIRAS_GlobalDevice, "GlobalDevice"); add(SPIRAS_GlobalHost, "GlobalHost"); } typedef SPIRVMap SPIRAddrSpaceCapitalizedNameMap; template <> inline void SPIRVMap::init() { add(SPIRAS_Private, StorageClassFunction); add(SPIRAS_Global, StorageClassCrossWorkgroup); add(SPIRAS_Constant, StorageClassUniformConstant); add(SPIRAS_Local, StorageClassWorkgroup); add(SPIRAS_Generic, StorageClassGeneric); add(SPIRAS_Input, StorageClassInput); add(SPIRAS_GlobalDevice, StorageClassDeviceOnlyINTEL); add(SPIRAS_GlobalHost, StorageClassHostOnlyINTEL); add(SPIRAS_CodeSectionINTEL, StorageClassCodeSectionINTEL); } typedef SPIRVMap SPIRSPIRVAddrSpaceMap; // Maps OCL builtin function to SPIRV builtin variable. template <> inline void SPIRVMap::init() { add("read_only", AccessQualifierReadOnly); add("write_only", AccessQualifierWriteOnly); add("read_write", AccessQualifierReadWrite); } typedef SPIRVMap SPIRSPIRVAccessQualifierMap; template <> inline void SPIRVMap::init() { add(Attribute::ZExt, FunctionParameterAttributeZext); add(Attribute::SExt, FunctionParameterAttributeSext); add(Attribute::ByVal, FunctionParameterAttributeByVal); add(Attribute::StructRet, FunctionParameterAttributeSret); add(Attribute::NoAlias, FunctionParameterAttributeNoAlias); add(Attribute::NoCapture, FunctionParameterAttributeNoCapture); add(Attribute::ReadOnly, FunctionParameterAttributeNoWrite); add(Attribute::ReadNone, FunctionParameterAttributeNoReadWrite); } typedef SPIRVMap SPIRSPIRVFuncParamAttrMap; template <> inline void SPIRVMap::init() { add(Attribute::ReadNone, FunctionControlPureMask); add(Attribute::ReadOnly, FunctionControlConstMask); add(Attribute::AlwaysInline, FunctionControlInlineMask); add(Attribute::NoInline, FunctionControlDontInlineMask); add(Attribute::OptimizeNone, FunctionControlOptNoneEXTMask); } typedef SPIRVMap SPIRSPIRVFuncCtlMaskMap; class SPIRVExtSetShortName; template <> inline void SPIRVMap::init() { add(SPIRVEIS_OpenCL, "ocl"); } typedef SPIRVMap SPIRVExtSetShortNameMap; template <> inline void SPIRVMap::init() { add(internal::RowMajor, "matrix.rowmajor"); add(internal::ColumnMajor, "matrix.columnmajor"); add(internal::PackedA, "matrix.packed.a"); add(internal::PackedB, "matrix.packed.b"); } typedef SPIRVMap SPIRVMatrixLayoutMap; template <> inline void SPIRVMap::init() { add(ScopeWorkgroup, "scope.workgroup"); add(ScopeSubgroup, "scope.subgroup"); } typedef SPIRVMap SPIRVMatrixScopeMap; #define SPIR_MD_COMPILER_OPTIONS "opencl.compiler.options" #define SPIR_MD_KERNEL_ARG_ADDR_SPACE "kernel_arg_addr_space" #define SPIR_MD_KERNEL_ARG_ACCESS_QUAL "kernel_arg_access_qual" #define SPIR_MD_KERNEL_ARG_TYPE "kernel_arg_type" #define SPIR_MD_KERNEL_ARG_BASE_TYPE "kernel_arg_base_type" #define SPIR_MD_KERNEL_ARG_TYPE_QUAL "kernel_arg_type_qual" #define SPIR_MD_KERNEL_ARG_NAME "kernel_arg_name" #define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations" #define SPIRV_MD_DECORATIONS "spirv.Decorations" #define OCL_TYPE_NAME_SAMPLER_T "sampler_t" #define SPIR_TYPE_NAME_EVENT_T "opencl.event_t" #define SPIR_TYPE_NAME_CLK_EVENT_T "opencl.clk_event_t" #define SPIR_TYPE_NAME_BLOCK_T "opencl.block" #define SPIR_INTRINSIC_BLOCK_BIND "spir_block_bind" #define SPIR_INTRINSIC_GET_BLOCK_INVOKE "spir_get_block_invoke" #define SPIR_INTRINSIC_GET_BLOCK_CONTEXT "spir_get_block_context" #define SPIR_TEMP_NAME_PREFIX_BLOCK "block" #define SPIR_TEMP_NAME_PREFIX_CALL "call" namespace kLLVMTypeName { const static char StructPrefix[] = "struct."; } // namespace kLLVMTypeName namespace kSPIRVImageSampledTypeName { const static char Float[] = "float"; const static char Half[] = "half"; const static char Int[] = "int"; const static char UInt[] = "uint"; const static char Long[] = "long"; const static char ULong[] = "ulong"; const static char Void[] = "void"; } // namespace kSPIRVImageSampledTypeName namespace kSPIRVTypeName { const static char Delimiter = '.'; const static char DeviceEvent[] = "DeviceEvent"; const static char Event[] = "Event"; const static char Image[] = "Image"; const static char Pipe[] = "Pipe"; const static char PostfixDelim = '_'; const static char Prefix[] = "spirv"; const static char PrefixAndDelim[] = "spirv."; const static char Queue[] = "Queue"; const static char ReserveId[] = "ReserveId"; const static char SampledImg[] = "SampledImage"; const static char Sampler[] = "Sampler"; const static char ConstantSampler[] = "ConstantSampler"; const static char PipeStorage[] = "PipeStorage"; const static char ConstantPipeStorage[] = "ConstantPipeStorage"; const static char VmeImageINTEL[] = "VmeImageINTEL"; const static char JointMatrixINTEL[] = "JointMatrixINTEL"; const static char CooperativeMatrixKHR[] = "CooperativeMatrixKHR"; } // namespace kSPIRVTypeName namespace kSPR2TypeName { const static char Delimiter = '.'; const static char OCLPrefix[] = "opencl."; const static char ImagePrefix[] = "opencl.image"; const static char PipeRO[] = "opencl.pipe_ro_t"; const static char PipeWO[] = "opencl.pipe_wo_t"; const static char Sampler[] = "opencl.sampler_t"; const static char Event[] = "opencl.event_t"; } // namespace kSPR2TypeName namespace kAccessQualName { const static char ReadOnly[] = "read_only"; const static char WriteOnly[] = "write_only"; const static char ReadWrite[] = "read_write"; } // namespace kAccessQualName namespace kAccessQualPostfix { const static char ReadOnly[] = "_ro"; const static char WriteOnly[] = "_wo"; const static char ReadWrite[] = "_rw"; const static char Type[] = "_t"; } // namespace kAccessQualPostfix namespace kMangledName { const static char Sampler[] = "11ocl_sampler"; const static char AtomicPrefixIncoming[] = "U7_Atomic"; const static char AtomicPrefixInternal[] = "atomic_"; } // namespace kMangledName namespace kSPIRVName { const static char GroupPrefix[] = "group_"; const static char GroupNonUniformPrefix[] = "group_non_uniform_"; const static char ClusteredPrefix[] = "clustered_"; const static char Prefix[] = "__spirv_"; const static char Postfix[] = "__"; const static char ImageQuerySize[] = "ImageQuerySize"; const static char ImageQuerySizeLod[] = "ImageQuerySizeLod"; const static char ImageSampleExplicitLod[] = "ImageSampleExplicitLod"; const static char ReservedPrefix[] = "reserved_"; const static char SampledImage[] = "SampledImage"; const static char TempSampledImage[] = "TempSampledImage"; const static char TranslateOCLMemOrder[] = "__translate_ocl_memory_order"; const static char TranslateOCLMemScope[] = "__translate_ocl_memory_scope"; const static char TranslateSPIRVMemOrder[] = "__translate_spirv_memory_order"; const static char TranslateSPIRVMemScope[] = "__translate_spirv_memory_scope"; const static char TranslateSPIRVMemFence[] = "__translate_spirv_memory_fence"; const static char EntrypointPrefix[] = "__spirv_entry_"; const static char ConvertHandleToImageINTEL[] = "ConvertHandleToImageINTEL"; const static char ConvertHandleToSamplerINTEL[] = "ConvertHandleToSamplerINTEL"; const static char ConvertHandleToSampledImageINTEL[] = "ConvertHandleToSampledImageINTEL"; } // namespace kSPIRVName namespace kSPIRVPostfix { const static char ToGlobal[] = "ToGlobal"; const static char ToLocal[] = "ToLocal"; const static char ToPrivate[] = "ToPrivate"; const static char Sat[] = "sat"; const static char Rtz[] = "rtz"; const static char Rte[] = "rte"; const static char Rtp[] = "rtp"; const static char Rtn[] = "rtn"; const static char Rt[] = "rt"; const static char Return[] = "R"; const static char Divider[] = "_"; /// Divider between extended instruction name and postfix const static char ExtDivider[] = "__"; } // namespace kSPIRVPostfix namespace kSPIRVMD { const static char Capability[] = "spirv.Capability"; const static char EntryPoint[] = "spirv.EntryPoint"; const static char ExecutionMode[] = "spirv.ExecutionMode"; const static char Extension[] = "spirv.Extension"; const static char Generator[] = "spirv.Generator"; const static char Source[] = "spirv.Source"; const static char SourceExtension[] = "spirv.SourceExtension"; const static char MemoryModel[] = "spirv.MemoryModel"; } // namespace kSPIRVMD namespace kSPIR2MD { const static char Extensions[] = "opencl.used.extensions"; const static char FPContract[] = "opencl.enable.FP_CONTRACT"; const static char OCLVer[] = "opencl.ocl.version"; const static char OptFeatures[] = "opencl.used.optional.core.features"; const static char SPIRVer[] = "opencl.spir.version"; const static char VecTyHint[] = "vec_type_hint"; const static char WGSize[] = "reqd_work_group_size"; const static char WGSizeHint[] = "work_group_size_hint"; const static char SubgroupSize[] = "intel_reqd_sub_group_size"; const static char MaxWGSize[] = "max_work_group_size"; const static char NoGlobalOffset[] = "no_global_work_offset"; const static char MaxWGDim[] = "max_global_work_dim"; const static char NumSIMD[] = "num_simd_work_items"; const static char StallEnable[] = "stall_enable"; const static char FmaxMhz[] = "scheduler_target_fmax_mhz"; const static char LoopFuse[] = "loop_fuse"; const static char PreferDSP[] = "prefer_dsp"; const static char PropDSPPref[] = "propagate_dsp_preference"; const static char InitiationInterval[] = "initiation_interval"; const static char MaxConcurrency[] = "max_concurrency"; const static char DisableLoopPipelining[] = "disable_loop_pipelining"; const static char IntelFPGAIPInterface[] = "ip_interface"; } // namespace kSPIR2MD enum Spir2SamplerKind { CLK_ADDRESS_NONE = 0x0000, CLK_ADDRESS_CLAMP = 0x0004, CLK_ADDRESS_CLAMP_TO_EDGE = 0x0002, CLK_ADDRESS_REPEAT = 0x0006, CLK_ADDRESS_MIRRORED_REPEAT = 0x0008, CLK_NORMALIZED_COORDS_FALSE = 0x0000, CLK_NORMALIZED_COORDS_TRUE = 0x0001, CLK_FILTER_NEAREST = 0x0010, CLK_FILTER_LINEAR = 0x0020, }; /// Additional information for mangling a function argument type. struct BuiltinArgTypeMangleInfo { bool IsSigned; bool IsVoidPtr; bool IsEnum; bool IsSampler; bool IsAtomic; bool IsLocalArgBlock; SPIR::TypePrimitiveEnum Enum; unsigned Attr; PointerIndirectPair PointerElementType; BuiltinArgTypeMangleInfo() : IsSigned(true), IsVoidPtr(false), IsEnum(false), IsSampler(false), IsAtomic(false), IsLocalArgBlock(false), Enum(SPIR::PRIMITIVE_NONE), Attr(0), PointerElementType(nullptr, false) {} }; /// Information for mangling builtin function. class BuiltinFuncMangleInfo { public: /// Translate builtin function name and set /// argument attributes and unsigned args. BuiltinFuncMangleInfo(const std::string &UniqName = "") : VarArgIdx(-1), DontMangle(false) { if (!UniqName.empty()) init(UniqName); } virtual ~BuiltinFuncMangleInfo() {} const std::string &getUnmangledName() const { return UnmangledName; } void addUnsignedArg(int Ndx) { if (Ndx == -1) return addUnsignedArgs(0, 10); // 10 is enough for everybody, right? getTypeMangleInfo(Ndx).IsSigned = false; } void addUnsignedArgs(int StartNdx, int StopNdx) { assert(StartNdx < StopNdx && "wrong parameters"); for (int I = StartNdx; I <= StopNdx; ++I) addUnsignedArg(I); } void addVoidPtrArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsVoidPtr = true; } void addSamplerArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsSampler = true; } void addAtomicArg(unsigned Ndx) { getTypeMangleInfo(Ndx).IsAtomic = true; } void setLocalArgBlock(unsigned Ndx) { getTypeMangleInfo(Ndx).IsLocalArgBlock = true; } void setEnumArg(unsigned Ndx, SPIR::TypePrimitiveEnum Enum) { auto &Info = getTypeMangleInfo(Ndx); Info.IsEnum = true; Info.Enum = Enum; } void setArgAttr(unsigned Ndx, unsigned Attr) { getTypeMangleInfo(Ndx).Attr = Attr; } void setVarArg(int Ndx) { assert(0 <= Ndx && "it is not allowed to set less than zero index"); VarArgIdx = Ndx; } void setAsDontMangle() { DontMangle = true; } bool avoidMangling() { return DontMangle; } // get ellipsis index, single ellipsis at the end of the function is possible // only return value < 0 if none int getVarArg() const { return VarArgIdx; } BuiltinArgTypeMangleInfo &getTypeMangleInfo(unsigned Ndx) { while (Ndx >= ArgInfo.size()) ArgInfo.emplace_back(); BuiltinArgTypeMangleInfo &Info = ArgInfo[Ndx]; return Info; } virtual void init(StringRef UniqUnmangledName) { UnmangledName = UniqUnmangledName.str(); } void fillPointerElementTypes(ArrayRef); protected: std::string UnmangledName; std::vector ArgInfo; int VarArgIdx; // index of ellipsis argument, idx < 0 if none private: bool DontMangle; // clang doesn't apply mangling for some builtin functions // (i.e. enqueue_kernel) }; /// \returns a vector of types for a collection of values. template std::vector getTypes(T V) { std::vector Tys; for (auto &I : V) Tys.push_back(I->getType()); return Tys; } /// Move elements of std::vector from [begin, end) to target. template void move(std::vector &V, size_t Begin, size_t End, size_t Target) { assert(Begin < End && End <= V.size() && Target <= V.size() && !(Begin < Target && Target < End)); if (Begin <= Target && Target <= End) return; auto B = V.begin() + Begin, E = V.begin() + End; if (Target > V.size()) Target = V.size(); if (Target > End) Target -= (End - Begin); std::vector Segment(B, E); V.erase(B, E); V.insert(V.begin() + Target, Segment.begin(), Segment.end()); } /// Find position of first pointer type value in a vector. inline size_t findFirstPtr(const std::vector &Args) { auto PtArg = std::find_if(Args.begin(), Args.end(), [](Value *V) { return V->getType()->isPointerTy(); }); return PtArg - Args.begin(); } bool isSupportedTriple(Triple T); void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr); void addFnAttr(CallInst *Call, Attribute::AttrKind Attr); void saveLLVMModule(Module *M, const std::string &OutputFile); std::string mapSPIRVTypeToOCLType(SPIRVType *Ty, bool Signed); std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed, Type *PointerElementType = nullptr); SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target); /// Return vector V extended with poison elements to match the number of /// components of NewType. Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder); /// Add decorations to a SPIR-V entry. /// \param Decs Each string is a postfix without _ at the beginning. SPIRVValue *addDecorations(SPIRVValue *Target, const SmallVectorImpl &Decs); PointerType *getOrCreateOpaquePtrType(Module *M, const std::string &Name, unsigned AddrSpace = SPIRAS_Global); StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name); PointerType *getSamplerType(Module *M); Type *getSamplerStructType(Module *M); PointerType *getSPIRVOpaquePtrType(Module *M, Op OC); void getFunctionTypeParameterTypes(llvm::FunctionType *FT, std::vector &ArgTys); Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef ArgTypes, StringRef Name, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); PointerType *getOCLClkEventType(Module *M); PointerType *getOCLClkEventPtrType(Module *M); Constant *getOCLNullClkEventPtr(Module *M); /// Get function call arguments. /// \param Start Starting index. /// \param End Ending index. std::vector getArguments(CallInst *CI, unsigned Start = 0, unsigned End = 0); /// Get constant function call argument as an integer. /// \param I argument index. uint64_t getArgAsInt(CallInst *CI, unsigned I); /// Get constant function call argument as type \param T. /// \param I argument index. template T getArgAs(CallInst *CI, unsigned I) { return static_cast(getArgAsInt(CI, I)); } /// Get constant function call argument as a Scope enum. /// \param I argument index. Scope getArgAsScope(CallInst *CI, unsigned I); /// Get constant function call argument as a Decoration enum. /// \param I argument index. Decoration getArgAsDecoration(CallInst *CI, unsigned I); /// Check if a type is SPIRV sampler type. bool isSPIRVSamplerType(llvm::Type *Ty); /// Check if a type is OCL image type (if pointed to). /// \return type name without "opencl." prefix. bool isOCLImageStructType(llvm::Type *Ty, StringRef *Name = nullptr); /// \param BaseTyName is the type name as in spirv.BaseTyName.Postfixes /// \param Postfix contains postfixes extracted from the SPIR-V image /// type name as spirv.BaseTyName.Postfixes. bool isSPIRVStructType(llvm::Type *Ty, StringRef BaseTyName, StringRef *Postfix = 0); bool isSYCLHalfType(llvm::Type *Ty); bool isSYCLBfloat16Type(llvm::Type *Ty); /// Decorate a function name as __spirv_{Name}_ std::string decorateSPIRVFunction(const std::string &S); /// Remove prefix/postfix from __spirv_{Name}_ StringRef undecorateSPIRVFunction(StringRef S); /// Check if a function has decorated name as __spirv_{Name}_ /// and get the original name. bool isDecoratedSPIRVFunc(const Function *F, StringRef &UndecName); std::string prefixSPIRVName(const std::string &S); StringRef dePrefixSPIRVName(StringRef R, SmallVectorImpl &Postfix); /// Get a canonical function name for a SPIR-V op code. std::string getSPIRVFuncName(Op OC, StringRef PostFix = ""); std::string getSPIRVFuncName(Op OC, const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); std::string getSPIRVFuncName(SPIRVBuiltinVariableKind BVKind); /// Get a canonical function name for a SPIR-V extended instruction std::string getSPIRVExtFuncName(SPIRVExtInstSetKind Set, unsigned ExtOp, StringRef PostFix = ""); /// Get SPIR-V op code given the canonical function name. /// Assume \param Name is either IA64 mangled or unmangled, and the unmangled /// name takes the __spirv_{OpName}_{Postfixes} format. /// \return op code if the unmangled function name is a valid op code name, /// otherwise return OpNop. /// \param Dec contains decorations decoded from function name if it is /// not nullptr. Op getSPIRVFuncOC(StringRef Name, SmallVectorImpl *Dec = nullptr); /// Get SPIR-V builtin variable enum given the canonical builtin name /// Assume \param Name is in format __spirv_BuiltIn{Name} /// \return false if \param Name is not a valid builtin name. bool getSPIRVBuiltin(const std::string &Name, spv::BuiltIn &Builtin); /// \param Name LLVM function name /// \param DemangledName demanged name of the OpenCL built-in function /// \returns true if Name is the name of the OpenCL built-in function, /// false for other functions bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp = false); /// Check if a function returns void bool isVoidFuncTy(FunctionType *FT); /// \returns true if function \p F has array type argument. bool hasArrayArg(Function *F); /// Mutates function call instruction by changing the arguments. /// \param ArgMutate mutates the function arguments. /// \return mutated call instruction. CallInst *mutateCallInst( Module *M, CallInst *CI, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutates function call instruction by changing the arguments and return /// value. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the return value. /// \return mutated instruction. Instruction *mutateCallInst( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = false); /// Mutate call instruction to call SPIR-V builtin function. CallInst *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &)> ArgMutate, AttributeList *Attrs = nullptr); /// Mutate call instruction to call SPIR-V builtin function. Instruction *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, AttributeList *Attrs = nullptr); /// Mutate function by change the arguments. /// \param ArgMutate mutates the function arguments. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Mutate function by change the arguments & the return type. /// \param ArgMutate mutates the function arguments. /// \param RetMutate mutates the function return value. /// \param TakeName Take the original function's name if a new function with /// different type needs to be created. void mutateFunction( Function *F, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle = nullptr, AttributeList *Attrs = nullptr, bool TakeName = true); /// Add a call instruction at \p Pos. CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, Instruction *Pos, BuiltinFuncMangleInfo *Mangle = nullptr, StringRef InstName = SPIR_TEMP_NAME_PREFIX_CALL, bool TakeFuncName = true); /// Add a call instruction for SPIR-V builtin function. CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, ArrayRef PointerElementTypes, Instruction *Pos, StringRef InstName); /// Add a call of spir_block_bind function. CallInst *addBlockBind(Module *M, Function *InvokeFunc, Value *BlkCtx, Value *CtxLen, Value *CtxAlign, Instruction *InsPos, StringRef InstName = SPIR_TEMP_NAME_PREFIX_BLOCK); typedef std::pair::iterator, std::vector::iterator> ValueVecRange; /// Add a vector at \param InsPos. Value *addVector(Instruction *InsPos, ValueVecRange Range); /// Replace scalar values with a vector created at \param InsPos. void makeVector(Instruction *InsPos, std::vector &Ops, ValueVecRange Range); /// Expand a vector type value in \param Ops at index \param VecPos. /// Generate extract element instructions at \param InsPos and replace /// the vector type value with scalar type values. /// If the value to be expanded is not vector type, do nothing. void expandVector(Instruction *InsPos, std::vector &Ops, size_t VecPos); /// Get size_t type. IntegerType *getSizetType(Module *M); /// Get void(void) function type. Type *getVoidFuncType(Module *M); /// Get void(void) function pointer type. Type *getVoidFuncPtrType(Module *M, unsigned AddrSpace = 0); /// Get a 64 bit integer constant. ConstantInt *getInt64(Module *M, int64_t Value); /// Get a 32 bit integer constant. ConstantInt *getInt32(Module *M, int Value); /// Get a 32 bit unsigned integer constant. ConstantInt *getUInt32(Module *M, unsigned Value); /// Get 32 bit integer constant if the value fits in 32 bits, /// return 64 bit integer constant otherwise ConstantInt *getInt(Module *M, int64_t Value); /// Get 32 bit unsigned integer constant if the value fits in 32 bits, /// return 64 bit unsigned integer constant otherwise ConstantInt *getUInt(Module *M, uint64_t Value); /// Get a 16 bit unsigned integer constant. ConstantInt *getUInt16(Module *M, unsigned short Value); // Get a 32 bit floating point constant. Constant *getFloat32(Module *M, float Value); /// Get a 32 bit integer constant vector. std::vector getInt32(Module *M, const std::vector &Value); /// Get a size_t type constant. ConstantInt *getSizet(Module *M, uint64_t Value); /// Get metadata operand as int. int64_t getMDOperandAsInt(MDNode *N, unsigned I); /// Get metadata operand as string. StringRef getMDOperandAsString(MDNode *N, unsigned I); /// Get metadata operand as another metadata node MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I); /// Get metadata operand as type. Type *getMDOperandAsType(MDNode *N, unsigned I); /// Get a named metadata as a set of string. /// Assume the named metadata has one or more operands each of which might /// contain set of strings. For instance: /// !opencl.used.optional.core.features = !{!0} /// !0 = !{!"cl_doubles", !"cl_images"} /// or if we linked two modules we may have /// !opencl.used.optional.core.features = !{!0, !1} /// !0 = !{!"cl_doubles"} /// !1 = !{!"cl_images"} std::set getNamedMDAsStringSet(Module *M, const std::string &MDName); /// Get SPIR-V language by SPIR-V metadata spirv.Source std::tuple getSPIRVSource(Module *M); /// Map an unsigned integer constant by applying a function. ConstantInt *mapUInt(Module *M, ConstantInt *I, std::function F); /// Map a signed integer constant by applying a function. ConstantInt *mapSInt(Module *M, ConstantInt *I, std::function F); /// Get postfix for given decoration. /// The returned postfix does not include "_" at the beginning. std::string getPostfix(Decoration Dec, unsigned Value = 0); /// Get postfix _R{ReturnType} for return type /// The returned postfix does not includ "_" at the beginning std::string getPostfixForReturnType(CallInst *CI, bool IsSigned = false); std::string getPostfixForReturnType(const Type *PRetTy, bool IsSigned = false, Type *PointerElementType = nullptr); Constant *getScalarOrVectorConstantInt(Type *T, uint64_t V, bool IsSigned = false); /// Get a constant int or a constant int array. /// \param T is the type of the constant. It should be an integer type or // an integer pointer type. /// \param Len is the length of the array. /// \param V is the value to fill the array. Value *getScalarOrArrayConstantInt(Instruction *P, Type *T, unsigned Len, uint64_t V, bool IsSigned = false); /// Get the array from GEP. /// \param V is a GEP whose pointer operand is a pointer to an array of size /// \param Size. Value *getScalarOrArray(Value *V, unsigned Size, Instruction *Pos); void dumpUsers(Value *V, StringRef Prompt = ""); /// Get SPIR-V type name as spirv.BaseTyName.Postfixes. std::string getSPIRVTypeName(StringRef BaseTyName, StringRef Postfixes = ""); /// Checks if given type name is either ConstantSampler or ConsantPipeStorage. bool isSPIRVConstantName(StringRef TyName); /// Get SPIR-V type by changing the type name from spirv.OldName.Postfixes /// to spirv.NewName.Postfixes. Type *getSPIRVTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName); /// Get SPIR-V type by changing the type name from spirv.OldName.Postfixes /// to spirv.NewName.Postfixes. Type *getSPIRVStructTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName); /// Get the postfixes of SPIR-V image type name as in spirv.Image.postfixes. std::string getSPIRVImageTypePostfixes(StringRef SampledType, SPIRVTypeImageDescriptor Desc, SPIRVAccessQualifierKind Acc); /// Get the sampled type name used in postfix of image type in SPIR-V /// friendly LLVM IR. std::string getSPIRVImageSampledTypeName(SPIRVType *Ty); /// Translates OpenCL image type names to SPIR-V. /// E.g. %opencl.image1d_rw_t -> %spirv.Image._void_0_0_0_0_0_0_2 Type *adaptSPIRVImageType(Module *M, Type *PointeeType); /// Get LLVM type for sampled type of SPIR-V image type by postfix. Type *getLLVMTypeForSPIRVImageSampledTypePostfix(StringRef Postfix, LLVMContext &Ctx); /// Return the unqualified and unsuffixed base name of an image type. /// E.g. opencl.image2d_ro_t.3 -> image2d_t std::string getImageBaseTypeName(StringRef Name); /// Map OpenCL opaque type name to SPIR-V type name. std::string mapOCLTypeNameToSPIRV(StringRef Name, StringRef Acc = ""); /// Return the index of image operands given an image op. size_t getImageOperandsIndex(Op OpCode); /// Check if access qualifier is encoded in the type name. bool hasAccessQualifiedName(StringRef TyName); /// Get access qualifier from the type name. SPIRVAccessQualifierKind getAccessQualifier(StringRef TyName); /// Get access qualifier from the type name. StringRef getAccessQualifierPostfix(SPIRVAccessQualifierKind Access); /// Get access qualifier from the type name. StringRef getAccessQualifierFullName(StringRef TyName); bool eraseUselessFunctions(Module *M); /// Erase a function if it is declaration, has internal linkage and has no use. bool eraseIfNoUse(Function *F); void eraseIfNoUse(Value *V); // Check if a mangled type name is unsigned bool isMangledTypeUnsigned(char Mangled); // Check if a mangled type name is signed bool isMangledTypeSigned(char Mangled); // Check if a mangled type name is floating point (except half) bool isMangledTypeFP(char Mangled); // Check if a mangled type name is half bool isMangledTypeHalf(std::string Mangled); // Check if \param I is valid vector size: 2, 3, 4, 8, 16. bool isValidVectorSize(unsigned I); enum class ParamType { FLOAT = 0, SIGNED = 1, UNSIGNED = 2, UNKNOWN = 3 }; ParamType lastFuncParamType(StringRef MangledName); // Check if the last function parameter is signed bool isLastFuncParamSigned(StringRef MangledName); // Check if a mangled function name contains unsigned atomic type bool containsUnsignedAtomicType(StringRef Name); /// Mangle builtin function name. /// \return \param UniqName if \param BtnInfo is null pointer, otherwise /// return IA64 mangled name. std::string mangleBuiltin(StringRef UniqName, ArrayRef ArgTypes, BuiltinFuncMangleInfo *BtnInfo); /// Extract the pointee types of arguments from a mangled function name. If the /// corresponding type is not a pointer to a struct type, its value will be a /// nullptr instead. void getParameterTypes(Function *F, SmallVectorImpl &ArgTys); inline void getParameterTypes(CallInst *CI, SmallVectorImpl &ArgTys) { return getParameterTypes(CI->getCalledFunction(), ArgTys); } /// Mangle a function from OpenCL extended instruction set in SPIR-V friendly IR /// manner std::string getSPIRVFriendlyIRFunctionName(OCLExtOpKind ExtOpId, ArrayRef ArgTys, ArrayRef PETs, Type *RetTy = nullptr); /// Mangle a function in SPIR-V friendly IR manner /// \param UniqName full unmangled name of the SPIR-V built-in function that /// contains possible postfixes that depend not on opcode but on decorations or /// return type, for example __spirv_UConvert_Rint_sat. /// \param OC opcode of corresponding built-in instruction. Used to gather info /// for unsigned/constant arguments. /// \param Types of arguments of SPIR-V built-in function /// \param Ops Operands of SPIRVInstruction /// \return IA64 mangled name. std::string getSPIRVFriendlyIRFunctionName(const std::string &UniqName, spv::Op OC, ArrayRef ArgTys, ArrayRef PETs, ArrayRef Ops); /// Cast a function to a void(void) funtion pointer. Constant *castToVoidFuncPtr(Function *F); /// Get i8* with the same address space. PointerType *getInt8PtrTy(PointerType *T); /// Cast a value to a i8* by inserting a cast instruction. Value *castToInt8Ptr(Value *V, Instruction *Pos); template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x) add(#x, OpType##x); _SPIRV_OP(DeviceEvent) _SPIRV_OP(Event) _SPIRV_OP(Image) _SPIRV_OP(Pipe) _SPIRV_OP(Queue) _SPIRV_OP(ReserveId) _SPIRV_OP(Sampler) _SPIRV_OP(SampledImage) // SPV_INTEL_device_side_avc_motion_estimation types _SPIRV_OP(AvcMcePayloadINTEL) _SPIRV_OP(AvcImePayloadINTEL) _SPIRV_OP(AvcRefPayloadINTEL) _SPIRV_OP(AvcSicPayloadINTEL) _SPIRV_OP(AvcMceResultINTEL) _SPIRV_OP(AvcImeResultINTEL) _SPIRV_OP(AvcImeResultSingleReferenceStreamoutINTEL) _SPIRV_OP(AvcImeResultDualReferenceStreamoutINTEL) _SPIRV_OP(AvcImeSingleReferenceStreaminINTEL) _SPIRV_OP(AvcImeDualReferenceStreaminINTEL) _SPIRV_OP(AvcRefResultINTEL) _SPIRV_OP(AvcSicResultINTEL) _SPIRV_OP(CooperativeMatrixKHR) #undef _SPIRV_OP } // Check if the module contains llvm.loop.* metadata bool hasLoopMetadata(const Module *M); // Check if CI is a call to instruction from OpenCL Extended Instruction Set. // If so, return it's extended opcode in ExtOp. bool isSPIRVOCLExtInst(const CallInst *CI, OCLExtOpKind *ExtOp); // check LLVM Intrinsics type(s) for validity bool checkTypeForSPIRVExtendedInstLowering(IntrinsicInst *II, SPIRVModule *BM); /// Decode SPIR-V type name in the format spirv.{TypeName}._{Postfixes} /// where Postfixes are strings separated by underscores. /// \return TypeName. /// \param Strs contains the integers decoded from postfixes. std::string decodeSPIRVTypeName(StringRef Name, SmallVectorImpl &Strs); // Copy attributes from function to call site. CallInst *setAttrByCalledFunc(CallInst *Call); bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind); // Transform builtin variable from GlobalVariable to builtin call. // e.g. // - GlobalInvolcationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x) // - WorkDim -> _Z22__spirv_BuiltInWorkDimv() bool lowerBuiltinVariableToCall(GlobalVariable *GV, SPIRVBuiltinVariableKind Kind); // Transform all builtin variables into calls bool lowerBuiltinVariablesToCalls(Module *M); // Transform all builtin calls into variables bool lowerBuiltinCallsToVariables(Module *M); // Transform all builtins into variables or calls // depending on user specification bool lowerBuiltins(SPIRVModule *BM, Module *M); /// \brief Post-process OpenCL or SPIRV builtin function returning struct type. /// /// Some builtin functions are translated to SPIR-V instructions with /// struct type result, e.g. NDRange creation functions. Such functions /// need to be post-processed to return the struct through sret argument. bool postProcessBuiltinReturningStruct(Function *F); /// \brief Post-process OpenCL or SPIRV builtin function having array argument. /// /// These functions are translated to functions with array type argument /// first, then post-processed to have pointer arguments. bool postProcessBuiltinWithArrayArguments(Function *F, StringRef DemangledName); bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp = false); bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp = false); template MetadataAsValue *map2MDString(LLVMContext &C, SPIRVValue *V); /// Returns the smallest integral power of two no smaller than Value if Value is /// nonzero. Returns 1 otherwise. /// /// Ex. bitCeil(5) == 8. /// /// The return value is undefined if the input is larger than the largest power /// of two representable in SPIRVWord. [[nodiscard]] SPIRVWord bitCeil(SPIRVWord Value); } // namespace SPIRV #endif // SPIRV_SPIRVINTERNAL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerBitCastToNonStandardType.cpp000066400000000000000000000234761506646577400263730ustar00rootroot00000000000000//===============- SPIRVLowerBitCastToNonStandardType.cpp -================// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2021 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering of BitCast to nonstandard types. LLVM // transformations bitcast some vector types to scalar types, which are not // universally supported across all targets. We need ensure that "optimized" // LLVM IR doesn't have primitive types other than supported by the // SPIR target (i.e. "scalar 8/16/32/64-bit integer and 16/32/64-bit floating // point types, 2/3/4/8/16-element vector of scalar types"). // //===----------------------------------------------------------------------===// #include "SPIRVLowerBitCastToNonStandardType.h" #include "SPIRVInternal.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/NoFolder.h" #include "llvm/Transforms/Utils/Local.h" #include #define DEBUG_TYPE "spv-lower-bitcast-to-nonstandard-type" using namespace llvm; namespace SPIRV { using NFIRBuilder = IRBuilder; static Value *removeBitCasts(Value *OldValue, Type *NewTy, NFIRBuilder &Builder, std::vector &InstsToErase) { IRBuilderBase::InsertPointGuard Guard(Builder); auto RauwBitcasts = [&](Instruction *OldValue, Value *NewValue) { // If there's only one use, don't create a bitcast for any uses, since it // will be immediately replaced anyways. if (OldValue->hasOneUse()) { OldValue->replaceAllUsesWith(UndefValue::get(OldValue->getType())); } else { OldValue->replaceAllUsesWith( Builder.CreateBitCast(NewValue, OldValue->getType())); } InstsToErase.push_back(OldValue); return NewValue; }; if (auto *LI = dyn_cast(OldValue)) { Builder.SetInsertPoint(LI); Value *Pointer = LI->getPointerOperand(); if (!Pointer->getType()->isOpaquePointerTy()) { Type *NewPointerTy = PointerType::get(NewTy, LI->getPointerAddressSpace()); Pointer = removeBitCasts(Pointer, NewPointerTy, Builder, InstsToErase); } LoadInst *NewLI = Builder.CreateAlignedLoad(NewTy, Pointer, LI->getAlign(), LI->isVolatile()); NewLI->setOrdering(LI->getOrdering()); NewLI->setSyncScopeID(LI->getSyncScopeID()); return RauwBitcasts(LI, NewLI); } if (auto *ASCI = dyn_cast(OldValue)) { Builder.SetInsertPoint(ASCI); Type *NewSrcTy = PointerType::getWithSamePointeeType( cast(NewTy), ASCI->getSrcAddressSpace()); Value *Pointer = removeBitCasts(ASCI->getPointerOperand(), NewSrcTy, Builder, InstsToErase); return RauwBitcasts(ASCI, Builder.CreateAddrSpaceCast(Pointer, NewTy)); } if (auto *BC = dyn_cast(OldValue)) { if (BC->getSrcTy() == NewTy) { if (BC->hasOneUse()) { BC->replaceAllUsesWith(UndefValue::get(BC->getType())); InstsToErase.push_back(BC); } return BC->getOperand(0); } Builder.SetInsertPoint(BC); return RauwBitcasts(BC, Builder.CreateBitCast(BC->getOperand(0), NewTy)); } report_fatal_error("Cannot translate source of bitcast instruction."); return nullptr; } static bool isNonStdVecType(VectorType *VecTy) { uint64_t NumElems = VecTy->getElementCount().getFixedValue(); return !isValidVectorSize(NumElems); } PreservedAnalyses SPIRVLowerBitCastToNonStandardTypePass::run(Function &F, FunctionAnalysisManager &FAM) { // This pass doesn't cover all possible uses of non-standard types, only // known. We assume that bad type won't be passed to a function as // parameter, since it added by an optimization. bool Changed = false; // SPV_INTEL_vector_compute allows to use vectors with any number of // components. Since this method only lowers vectors with non-standard // in pure SPIR-V number of components, there is no need to do anything in // case SPV_INTEL_vector_compute is enabled. if (Opts.isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) return PreservedAnalyses::all(); // The basic pattern we're trying to fix is this InstCombine pattern: // trunc (extractelement) -> extractelement (bitcast) // (note that the bitcast itself can get propagated back to change the type // of load instructions, and even through those to pointer casts, if typed // pointers are enabled. std::vector NonStdVecInsts; SmallVector MaybeDeletedInsts; for (auto &BB : F) for (auto &I : BB) { if (auto *EI = dyn_cast(&I)) { if (isNonStdVecType(EI->getVectorOperandType())) NonStdVecInsts.push_back(EI); } else if (auto *VT = dyn_cast(I.getType())) { if (isNonStdVecType(VT)) { MaybeDeletedInsts.push_back(&I); } } } std::vector InstsToErase; NFIRBuilder Builder(F.getContext()); for (auto &I : NonStdVecInsts) { VectorType *OldVecTy = I->getVectorOperandType(); unsigned OldVecSize = OldVecTy->getElementCount().getFixedValue(); // Compute the adjustment factor for the new vector size. unsigned VecFactor = 2; while (OldVecSize % VecFactor == 0 && !isValidVectorSize(OldVecSize / VecFactor)) VecFactor *= 2; if (OldVecSize % VecFactor != 0) { report_fatal_error(Twine("Invalid vector size for fixup: ") + Twine(OldVecSize)); return PreservedAnalyses::none(); } unsigned NewElemSize = OldVecTy->getScalarSizeInBits() * VecFactor; VectorType *NewVecTy = VectorType::get(Type::getIntNTy(F.getContext(), NewElemSize), OldVecSize / VecFactor, false); // Adjust the element index as appropriate. uint64_t OldElemIdx = cast(I->getIndexOperand())->getZExtValue(); uint64_t NewElemIdx = OldElemIdx / VecFactor; uint64_t ShiftCount = OldElemIdx % VecFactor; Builder.SetInsertPoint(I); Value *NewVecOp = removeBitCasts(I->getVectorOperand(), NewVecTy, Builder, InstsToErase); Value *NewExtracted = Builder.CreateExtractElement(NewVecOp, NewElemIdx); // If the extract does higher-order bits of the value, shift as necessary. if (ShiftCount > 0) NewExtracted = Builder.CreateLShr( NewExtracted, ShiftCount * OldVecTy->getScalarSizeInBits()); Value *NewValue = Builder.CreateTrunc(NewExtracted, I->getType()); I->replaceAllUsesWith(NewValue); I->eraseFromParent(); Changed = true; } for (auto *I : InstsToErase) RecursivelyDeleteTriviallyDeadInstructions(I); // Check if there are any residual unsupported vector types. for (auto &VH : MaybeDeletedInsts) { // Some vector-valued instructions were replaced with undef values, so if // that's what we got, it's still a dead instruction. if (VH.pointsToAliveValue() && !isa(VH)) { auto *VT = dyn_cast(VH->getType()); report_fatal_error(Twine("Unsupported vector type with ") + Twine(VT->getElementCount().getFixedValue()) + Twine(" elements"), false); } } return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); } bool SPIRVLowerBitCastToNonStandardTypeLegacy::runOnFunction(Function &F) { SPIRVLowerBitCastToNonStandardTypePass Impl(Opts); FunctionAnalysisManager FAM; auto PA = Impl.run(F, FAM); return !PA.areAllPreserved(); } bool SPIRVLowerBitCastToNonStandardTypeLegacy::doFinalization(Module &M) { verifyRegularizationPass(M, "SPIRVLowerBitCastToNonStandardType"); return false; } StringRef SPIRVLowerBitCastToNonStandardTypeLegacy::getPassName() const { return "Lower nonstandard type"; } char SPIRVLowerBitCastToNonStandardTypeLegacy::ID = 0; } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerBitCastToNonStandardTypeLegacy, "spv-lower-bitcast-to-nonstandard-type", "Remove bitcast to nonstandard types", false, false) llvm::FunctionPass *llvm::createSPIRVLowerBitCastToNonStandardTypeLegacy( const SPIRV::TranslatorOpts &Opts) { return new SPIRVLowerBitCastToNonStandardTypeLegacy(Opts); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerBitCastToNonStandardType.h000066400000000000000000000057211506646577400260310ustar00rootroot00000000000000//===- SPIRVLowerBitCastToNonStandardType.h - Bitcast lowering --*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVLOWERBITCASTTONONSTANDARDTYPE_H #define SPIRV_SPIRVLOWERBITCASTTONONSTANDARDTYPE_H #include "LLVMSPIRVOpts.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVLowerBitCastToNonStandardTypePass : public llvm::PassInfoMixin { public: SPIRVLowerBitCastToNonStandardTypePass(const SPIRV::TranslatorOpts &Opts) : Opts(Opts) {} llvm::PreservedAnalyses run(llvm::Function &F, llvm::FunctionAnalysisManager &FAM); private: SPIRV::TranslatorOpts Opts; }; class SPIRVLowerBitCastToNonStandardTypeLegacy : public llvm::FunctionPass { public: static char ID; SPIRVLowerBitCastToNonStandardTypeLegacy(const SPIRV::TranslatorOpts &Opts) : FunctionPass(ID), Opts(Opts) {} SPIRVLowerBitCastToNonStandardTypeLegacy() : FunctionPass(ID) {} bool runOnFunction(llvm::Function &F) override; bool doFinalization(llvm::Module &M) override; llvm::StringRef getPassName() const override; private: SPIRV::TranslatorOpts Opts; }; } // namespace SPIRV #endif // SPIRV_SPIRVLOWERBITCASTTONONSTANDARDTYPE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerBool.cpp000066400000000000000000000125261506646577400224260ustar00rootroot00000000000000//===- SPIRVLowerBool.cpp - Lower instructions with bool operands ---------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering instructions with bool operands. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spvbool" #include "SPIRVLowerBool.h" #include "SPIRVInternal.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/IR/IRBuilder.h" using namespace llvm; using namespace SPIRV; namespace SPIRV { void SPIRVLowerBoolBase::replace(Instruction *I, Instruction *NewI) { NewI->takeName(I); NewI->setDebugLoc(I->getDebugLoc()); I->replaceAllUsesWith(NewI); I->dropAllReferences(); I->eraseFromParent(); } bool SPIRVLowerBoolBase::isBoolType(Type *Ty) { if (Ty->isIntegerTy(1)) return true; if (auto VT = dyn_cast(Ty)) return isBoolType(VT->getElementType()); return false; } void SPIRVLowerBoolBase::visitTruncInst(TruncInst &I) { if (isBoolType(I.getType())) { auto Op = I.getOperand(0); auto And = BinaryOperator::CreateAnd( Op, getScalarOrVectorConstantInt(Op->getType(), 1, false), "", &I); And->setDebugLoc(I.getDebugLoc()); auto Zero = getScalarOrVectorConstantInt(Op->getType(), 0, false); auto Cmp = new ICmpInst(&I, CmpInst::ICMP_NE, And, Zero); replace(&I, Cmp); } } void SPIRVLowerBoolBase::handleExtInstructions(Instruction &I) { auto Op = I.getOperand(0); if (isBoolType(Op->getType())) { auto Opcode = I.getOpcode(); auto Ty = I.getType(); auto Zero = getScalarOrVectorConstantInt(Ty, 0, false); auto One = getScalarOrVectorConstantInt( Ty, (Opcode == Instruction::SExt) ? ~0 : 1, false); assert(Zero && One && "Couldn't create constant int"); auto Sel = SelectInst::Create(Op, One, Zero, "", &I); replace(&I, Sel); } } void SPIRVLowerBoolBase::handleCastInstructions(Instruction &I) { auto Op = I.getOperand(0); auto *OpTy = Op->getType(); if (isBoolType(OpTy)) { Type *Ty = Type::getInt32Ty(*Context); if (auto VT = dyn_cast(OpTy)) Ty = llvm::FixedVectorType::get(Ty, VT->getNumElements()); auto Zero = getScalarOrVectorConstantInt(Ty, 0, false); auto One = getScalarOrVectorConstantInt(Ty, 1, false); assert(Zero && One && "Couldn't create constant int"); auto Sel = SelectInst::Create(Op, One, Zero, "", &I); Sel->setDebugLoc(I.getDebugLoc()); I.setOperand(0, Sel); } } void SPIRVLowerBoolBase::visitZExtInst(ZExtInst &I) { handleExtInstructions(I); } void SPIRVLowerBoolBase::visitSExtInst(SExtInst &I) { handleExtInstructions(I); } void SPIRVLowerBoolBase::visitUIToFPInst(UIToFPInst &I) { handleCastInstructions(I); } void SPIRVLowerBoolBase::visitSIToFPInst(SIToFPInst &I) { handleCastInstructions(I); } bool SPIRVLowerBoolBase::runLowerBool(Module &M) { Context = &M.getContext(); visit(M); verifyRegularizationPass(M, "SPIRVLowerBool"); return true; } llvm::PreservedAnalyses SPIRVLowerBoolPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runLowerBool(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } SPIRVLowerBoolLegacy::SPIRVLowerBoolLegacy() : ModulePass(ID) { initializeSPIRVLowerBoolLegacyPass(*PassRegistry::getPassRegistry()); } bool SPIRVLowerBoolLegacy::runOnModule(Module &M) { return runLowerBool(M); } char SPIRVLowerBoolLegacy::ID = 0; } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerBoolLegacy, "spvbool", "Lower instructions with bool operands", false, false) ModulePass *llvm::createSPIRVLowerBoolLegacy() { return new SPIRVLowerBoolLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerBool.h000066400000000000000000000065711506646577400220760ustar00rootroot00000000000000//===- SPIRVLowerBool.h - Bool operand lowering --------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering instructions with bool operands. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVLOWERBOOL_H #define SPIRV_SPIRVLOWERBOOL_H #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVLowerBoolBase : public llvm::InstVisitor { public: SPIRVLowerBoolBase() : Context(nullptr) {} virtual ~SPIRVLowerBoolBase() {} void replace(llvm::Instruction *I, llvm::Instruction *NewI); bool isBoolType(llvm::Type *Ty); virtual void visitTruncInst(llvm::TruncInst &I); void handleExtInstructions(llvm::Instruction &I); void handleCastInstructions(llvm::Instruction &I); virtual void visitZExtInst(llvm::ZExtInst &I); virtual void visitSExtInst(llvm::SExtInst &I); virtual void visitUIToFPInst(llvm::UIToFPInst &I); virtual void visitSIToFPInst(llvm::SIToFPInst &I); bool runLowerBool(llvm::Module &M); private: llvm::LLVMContext *Context; }; class SPIRVLowerBoolPass : public llvm::PassInfoMixin, public SPIRVLowerBoolBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; class SPIRVLowerBoolLegacy : public llvm::ModulePass, public SPIRVLowerBoolBase { public: SPIRVLowerBoolLegacy(); bool runOnModule(llvm::Module &M) override; static char ID; }; } // namespace SPIRV #endif // SPIRV_SPIRVLOWERBOOL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerConstExpr.cpp000066400000000000000000000152771506646577400234660ustar00rootroot00000000000000//===- SPIRVLowerConstExpr.cpp - Regularize LLVM for SPIR-V ------- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements regularization of LLVM module for SPIR-V. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spv-lower-const-expr" #include "SPIRVLowerConstExpr.h" #include "OCLUtil.h" #include "SPIRVInternal.h" #include "SPIRVMDBuilder.h" #include "SPIRVMDWalker.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include #include using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { cl::opt SPIRVLowerConst( "spirv-lower-const-expr", cl::init(true), cl::desc("LLVM/SPIR-V translation enable lowering constant expression")); class SPIRVLowerConstExprLegacy : public ModulePass, public SPIRVLowerConstExprBase { public: SPIRVLowerConstExprLegacy() : ModulePass(ID) { initializeSPIRVLowerConstExprLegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override { return runLowerConstExpr(M); } static char ID; }; char SPIRVLowerConstExprLegacy::ID = 0; bool SPIRVLowerConstExprBase::runLowerConstExpr(Module &Module) { if (!SPIRVLowerConst) return false; M = &Module; Ctx = &M->getContext(); LLVM_DEBUG(dbgs() << "Enter SPIRVLowerConstExpr:\n"); bool Changed = visit(M); verifyRegularizationPass(*M, "SPIRVLowerConstExpr"); return Changed; } /// Since SPIR-V cannot represent constant expression, constant expressions /// in LLVM needs to be lowered to instructions. /// For each function, the constant expressions used by instructions of the /// function are replaced by instructions placed in the entry block since it /// dominates all other BB's. Each constant expression only needs to be lowered /// once in each function and all uses of it by instructions in that function /// is replaced by one instruction. /// ToDo: remove redundant instructions for common subexpression bool SPIRVLowerConstExprBase::visit(Module *M) { bool Changed = false; for (auto &I : M->functions()) { std::list WorkList; for (auto &BI : I) { for (auto &II : BI) { WorkList.push_back(&II); } } auto FBegin = I.begin(); while (!WorkList.empty()) { auto II = WorkList.front(); auto LowerOp = [&II, &FBegin, &I, &Changed](Value *V) -> Value * { if (isa(V)) return V; auto *CE = cast(V); SPIRVDBG(dbgs() << "[lowerConstantExpressions] " << *CE;) auto ReplInst = CE->getAsInstruction(); auto InsPoint = II->getParent() == &*FBegin ? II : &FBegin->back(); ReplInst->insertBefore(InsPoint); SPIRVDBG(dbgs() << " -> " << *ReplInst << '\n';) std::vector Users; // Do not replace use during iteration of use. Do it in another loop for (auto U : CE->users()) { SPIRVDBG(dbgs() << "[lowerConstantExpressions] Use: " << *U << '\n';) if (auto InstUser = dyn_cast(U)) { // Only replace users in scope of current function if (InstUser->getParent()->getParent() == &I) Users.push_back(InstUser); } } for (auto &User : Users) { if (ReplInst->getParent() == User->getParent()) if (User->comesBefore(ReplInst)) ReplInst->moveBefore(User); User->replaceUsesOfWith(CE, ReplInst); } Changed = true; return ReplInst; }; WorkList.pop_front(); for (unsigned OI = 0, OE = II->getNumOperands(); OI != OE; ++OI) { auto *Op = II->getOperand(OI); if (auto *CE = dyn_cast(Op)) { WorkList.push_front(cast(LowerOp(CE))); } else if (auto MDAsVal = dyn_cast(Op)) { Metadata *MD = MDAsVal->getMetadata(); if (auto ConstMD = dyn_cast(MD)) { Constant *C = ConstMD->getValue(); Value *ReplInst = nullptr; if (auto *CE = dyn_cast(C)) ReplInst = LowerOp(CE); if (ReplInst) { Metadata *RepMD = ValueAsMetadata::get(ReplInst); Value *RepMDVal = MetadataAsValue::get(M->getContext(), RepMD); II->setOperand(OI, RepMDVal); WorkList.push_front(cast(ReplInst)); } } } } } } return Changed; } } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerConstExprLegacy, "spv-lower-const-expr", "Regularize LLVM for SPIR-V", false, false) ModulePass *llvm::createSPIRVLowerConstExprLegacy() { return new SPIRVLowerConstExprLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerConstExpr.h000066400000000000000000000024671506646577400231300ustar00rootroot00000000000000//===- SPIRVLowerConstExpr.h - Lower constant expression --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// \file SPIRVLowerConstExpr.h /// /// This file declares SPIRVLowerConstExprPass that lowers constant expression. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LOWERCONSTEXPR_H #define SPIRV_LOWERCONSTEXPR_H #include "llvm/IR/PassManager.h" namespace SPIRV { class SPIRVLowerConstExprBase { public: SPIRVLowerConstExprBase() : M(nullptr), Ctx(nullptr) {} bool runLowerConstExpr(llvm::Module &M); bool visit(llvm::Module *M); private: llvm::Module *M; llvm::LLVMContext *Ctx; }; class SPIRVLowerConstExprPass : public llvm::PassInfoMixin, public SPIRVLowerConstExprBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runLowerConstExpr(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } }; } // namespace SPIRV #endif // SPIRV_LOWERCONSTEXPR_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerMemmove.cpp000066400000000000000000000132741506646577400231410ustar00rootroot00000000000000//===- SPIRVLowerMemmove.cpp - Lower llvm.memmove to llvm.memcpys ---------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering llvm.memmove into several llvm.memcpys. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spvmemmove" #include "SPIRVLowerMemmove.h" #include "SPIRVInternal.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" using namespace llvm; using namespace SPIRV; namespace SPIRV { void SPIRVLowerMemmoveBase::LowerMemMoveInst(MemMoveInst &I) { // There is no direct equivalent of @llvm.memmove in SPIR-V and the closest // instructions are 'OpCopyMemory' and 'OpCopyMemorySized'. // // 'OpCopyMemory' does not accept amount of bytes to copy and infers that // from type which is being copied; also it only allows to copy value of a // particular type to pointer pointing to the same type. // // 'OpCopyMemorySized' is closer to @llvm.memmove, because it actually // copies bytes, but unlike memove it is not explicitly specified whether it // supports overlapping source and destination. Therefore, we replace // memmove with two 'OpCopyMemorySized' instructions: the first one copies // bytes from source to a temporary location, the second one copies bytes // from that temporary location to the destination. IRBuilder<> Builder(I.getParent()); Builder.SetInsertPoint(&I); auto *Length = cast(I.getLength()); auto *AllocaTy = ArrayType::get(IntegerType::getInt8Ty(*Context), Length->getZExtValue()); MaybeAlign SrcAlign = I.getSourceAlign(); auto *Alloca = Builder.CreateAlloca(AllocaTy); if (SrcAlign.hasValue()) Alloca->setAlignment(SrcAlign.getValue()); // FIXME: Do we need to pass the size of alloca here? From LangRef: // > The first argument is a constant integer representing the size of the // > object, or -1 if it is variable sized. // // https://llvm.org/docs/LangRef.html#llvm-lifetime-start-intrinsic Builder.CreateLifetimeStart(Alloca); Builder.CreateMemCpy(Alloca, SrcAlign, I.getRawSource(), SrcAlign, Length, I.isVolatile()); auto *SecondCpy = Builder.CreateMemCpy(I.getRawDest(), I.getDestAlign(), Alloca, SrcAlign, Length, I.isVolatile()); Builder.CreateLifetimeEnd(Alloca); SecondCpy->takeName(&I); I.replaceAllUsesWith(SecondCpy); I.dropAllReferences(); I.eraseFromParent(); } bool SPIRVLowerMemmoveBase::expandMemMoveIntrinsicUses(Function &F) { bool Changed = false; for (User *U : make_early_inc_range(F.users())) { MemMoveInst *Inst = cast(U); if (!isa(Inst->getLength())) { expandMemMoveAsLoop(Inst); Inst->eraseFromParent(); } else { LowerMemMoveInst(*Inst); } Changed = true; } return Changed; } bool SPIRVLowerMemmoveBase::runLowerMemmove(Module &M) { Context = &M.getContext(); bool Changed = false; for (Function &F : M) { if (!F.isDeclaration()) continue; if (F.getIntrinsicID() == Intrinsic::memmove) Changed |= expandMemMoveIntrinsicUses(F); } verifyRegularizationPass(M, "SPIRVLowerMemmove"); return Changed; } llvm::PreservedAnalyses SPIRVLowerMemmovePass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runLowerMemmove(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } SPIRVLowerMemmoveLegacy::SPIRVLowerMemmoveLegacy() : ModulePass(ID) { initializeSPIRVLowerMemmoveLegacyPass(*PassRegistry::getPassRegistry()); } bool SPIRVLowerMemmoveLegacy::runOnModule(Module &M) { return runLowerMemmove(M); } char SPIRVLowerMemmoveLegacy::ID = 0; } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerMemmoveLegacy, "spvmemmove", "Lower llvm.memmove into llvm.memcpy", false, false) ModulePass *llvm::createSPIRVLowerMemmoveLegacy() { return new SPIRVLowerMemmoveLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerMemmove.h000066400000000000000000000057561506646577400226140ustar00rootroot00000000000000//===- SPIRVLowerMemmove.h - memmove lowering --------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering llvm.memmove into several llvm.memcpys. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVLOWERMEMMOVE_H #define SPIRV_SPIRVLOWERMEMMOVE_H #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVLowerMemmoveBase { public: SPIRVLowerMemmoveBase() : Context(nullptr) {} void LowerMemMoveInst(llvm::MemMoveInst &I); bool expandMemMoveIntrinsicUses(llvm::Function &F); bool runLowerMemmove(llvm::Module &M); private: llvm::LLVMContext *Context; }; class SPIRVLowerMemmovePass : public llvm::PassInfoMixin, public SPIRVLowerMemmoveBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; class SPIRVLowerMemmoveLegacy : public llvm::ModulePass, public SPIRVLowerMemmoveBase { public: SPIRVLowerMemmoveLegacy(); bool runOnModule(llvm::Module &M) override; static char ID; }; } // namespace SPIRV #endif // SPIRV_SPIRVLOWERMEMMOVE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerOCLBlocks.cpp000066400000000000000000000107231506646577400233030ustar00rootroot00000000000000//===- SPIRVLowerOCLBlocks.cpp - OCL Utilities ----------------------------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2018 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // SPIR-V specification doesn't allow function pointers, so SPIR-V translator // is designed to fail if a value with function type (except calls) occurs. // Currently there is only two cases, when function pointers are generating in // LLVM IR in OpenCL - block calls and device side enqueue built-in calls. // // In both cases values with function type used as intermediate representation // for block literal structure. // // In LLVM IR produced by clang, blocks are represented with the following // structure: // %struct.__opencl_block_literal_generic = type { i32, i32, i8 addrspace(4)* } // Pointers to block invoke functions are stored in the third field. Clang // replaces indirect function calls in all cases except if block is passed as a // function argument. Note that it is somewhat unclear if the OpenCL C spec // should allow passing blocks as function arguments. This pass is not supposed // to work correctly with such functions. // Clang though has to store function pointers to this structure. Purpose of // this pass is to replace store of function pointers(not allowed in SPIR-V) // with null pointers. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spv-lower-ocl-blocks" #include "SPIRVLowerOCLBlocks.h" #include "SPIRVInternal.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Support/Regex.h" using namespace llvm; namespace { static bool isBlockInvoke(Function &F) { static Regex BlockInvokeRegex("_block_invoke_?[0-9]*$"); return BlockInvokeRegex.match(F.getName()); } } // namespace namespace SPIRV { bool SPIRVLowerOCLBlocksBase::runLowerOCLBlocks(Module &M) { bool Changed = false; for (Function &F : M) { if (!isBlockInvoke(F)) continue; for (User *U : F.users()) { if (!isa(U)) continue; Constant *Null = Constant::getNullValue(U->getType()); if (U != Null) { U->replaceAllUsesWith(Null); Changed = true; } } } return Changed; } llvm::PreservedAnalyses SPIRVLowerOCLBlocksPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runLowerOCLBlocks(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } char SPIRVLowerOCLBlocksLegacy::ID = 0; } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerOCLBlocksLegacy, "spv-lower-ocl-blocks", "Remove function pointers originating from OpenCL blocks", false, false) llvm::ModulePass *llvm::createSPIRVLowerOCLBlocksLegacy() { return new SPIRVLowerOCLBlocksLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerOCLBlocks.h000066400000000000000000000054231506646577400227510ustar00rootroot00000000000000//===- SPIRVLowerOCLBlocks.h - OpenCL block lowering --------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVLOWEROCLBLOCKS_H #define SPIRV_SPIRVLOWEROCLBLOCKS_H #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVLowerOCLBlocksBase { public: SPIRVLowerOCLBlocksBase() {} bool runLowerOCLBlocks(llvm::Module &M); }; class SPIRVLowerOCLBlocksPass : public llvm::PassInfoMixin, public SPIRVLowerOCLBlocksBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; class SPIRVLowerOCLBlocksLegacy : public llvm::ModulePass, public SPIRVLowerOCLBlocksBase { public: SPIRVLowerOCLBlocksLegacy() : ModulePass(ID) {} bool runOnModule(llvm::Module &M) override { return runLowerOCLBlocks(M); } llvm::StringRef getPassName() const override { return "Lower OpenCL Blocks For SPIR-V"; } static char ID; }; } // namespace SPIRV #endif // SPIRV_SPIRVLOWEROCLBLOCKS_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerSaddIntrinsics.cpp000066400000000000000000000162041506646577400244510ustar00rootroot00000000000000//===- SPIRVLowerSaddIntrinsics.cpp - Lower llvm.sadd.* -------------------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2020 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements lowering of llvm.sadd.* into basic LLVM // operations. Probably, in the future this pass can be generalized for other // function calls // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spv-lower-llvm_sadd_intrinsics" #include "SPIRVLowerSaddIntrinsics.h" #include "LLVMSaddWithOverflow.h" #include "LLVMSPIRVLib.h" #include "SPIRVError.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" #include "llvm/Linker/Linker.h" #include "llvm/Support/SourceMgr.h" using namespace llvm; using namespace SPIRV; namespace SPIRV { void SPIRVLowerSaddIntrinsicsBase::replaceSaddOverflow(Function &F) { assert(F.getIntrinsicID() == Intrinsic::sadd_with_overflow); StringRef IntrinsicName = F.getName(); std::string FuncName = "llvm_sadd_with_overflow_i"; if (IntrinsicName.endswith(".i16")) FuncName += "16"; else if (IntrinsicName.endswith(".i32")) FuncName += "32"; else if (IntrinsicName.endswith(".i64")) FuncName += "64"; else { assert(false && "Unsupported overloading of llvm.sadd.with.overflow intrinsic"); return; } // Redirect @llvm.sadd.with.overflow.* call to the function we have in // the loaded module @llvm_sadd_with_overflow_* Function *ReplacementFunc = Mod->getFunction(FuncName); if (!ReplacementFunc) { // This function needs linking. Mod->getOrInsertFunction(FuncName, F.getFunctionType()); // Read LLVM IR with the intrinsic's implementation SMDiagnostic Err; auto MB = MemoryBuffer::getMemBuffer(LLVMSaddWithOverflow); auto SaddWithOverflowModule = parseIR(MB->getMemBufferRef(), Err, *Context, [&](StringRef) { return Mod->getDataLayoutStr(); }); if (!SaddWithOverflowModule) { std::string ErrMsg; raw_string_ostream ErrStream(ErrMsg); Err.print("", ErrStream); SPIRVErrorLog EL; EL.checkError(false, SPIRVEC_InvalidLlvmModule, ErrMsg); return; } // Link in the intrinsic's implementation. if (!Linker::linkModules(*Mod, std::move(SaddWithOverflowModule), Linker::LinkOnlyNeeded)) TheModuleIsModified = true; ReplacementFunc = Mod->getFunction(FuncName); assert(ReplacementFunc && "How did we not link in the necessary function?"); } F.replaceAllUsesWith(ReplacementFunc); } void SPIRVLowerSaddIntrinsicsBase::replaceSaddSat(Function &F) { assert(F.getIntrinsicID() == Intrinsic::sadd_sat); SmallVector Intrinsics; for (User *U : F.users()) { if (auto *II = dyn_cast(U)) Intrinsics.push_back(II); } // Get the corresponding sadd_with_overflow intrinsic for the sadd_sat. Type *IntTy = F.getFunctionType()->getReturnType(); Function *SaddO = Intrinsic::getDeclaration(Mod, Intrinsic::sadd_with_overflow, IntTy); // Replace all uses of the intrinsic with equivalent code relying on // sadd_with_overflow IRBuilder<> Builder(F.getContext()); unsigned BitWidth = IntTy->getIntegerBitWidth(); Value *IntMin = Builder.getInt(APInt::getSignedMinValue(BitWidth)); Value *ShiftWidth = Builder.getIntN(BitWidth, BitWidth - 1); for (IntrinsicInst *II : Intrinsics) { Builder.SetInsertPoint(II); // {res, overflow} = @llvm.sadd_with_overflow(a, b) // sadd_sat(a, b) => overflow ? (res >> bitwidth) ^ intmin : res; Value *StructRes = Builder.CreateCall(SaddO, {II->getArgOperand(0), II->getArgOperand(1)}); Value *Sum = Builder.CreateExtractValue(StructRes, 0); Value *Overflow = Builder.CreateExtractValue(StructRes, 1); Value *OverflowedRes = Builder.CreateXor(Builder.CreateAShr(Sum, ShiftWidth), IntMin); Value *Result = Builder.CreateSelect(Overflow, OverflowedRes, Sum); II->replaceAllUsesWith(Result); II->eraseFromParent(); } // Now replace the sadd_with_overflow intrinsic itself. replaceSaddOverflow(*SaddO); } bool SPIRVLowerSaddIntrinsicsBase::runLowerSaddIntrinsics(Module &M) { Context = &M.getContext(); Mod = &M; for (Function &F : M) { Intrinsic::ID IntrinId = F.getIntrinsicID(); if (IntrinId == Intrinsic::sadd_with_overflow) replaceSaddOverflow(F); else if (IntrinId == Intrinsic::sadd_sat) replaceSaddSat(F); } verifyRegularizationPass(M, "SPIRVLowerSaddIntrinsics"); return TheModuleIsModified; } llvm::PreservedAnalyses SPIRVLowerSaddIntrinsicsPass::run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runLowerSaddIntrinsics(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } SPIRVLowerSaddIntrinsicsLegacy::SPIRVLowerSaddIntrinsicsLegacy() : ModulePass(ID) { initializeSPIRVLowerSaddIntrinsicsLegacyPass( *PassRegistry::getPassRegistry()); } bool SPIRVLowerSaddIntrinsicsLegacy::runOnModule(Module &M) { return runLowerSaddIntrinsics(M); } char SPIRVLowerSaddIntrinsicsLegacy::ID = 0; } // namespace SPIRV INITIALIZE_PASS(SPIRVLowerSaddIntrinsicsLegacy, "spv-lower-llvm_sadd_intrinsics", "Lower llvm.sadd.* intrinsics", false, false) ModulePass *llvm::createSPIRVLowerSaddIntrinsicsLegacy() { return new SPIRVLowerSaddIntrinsicsLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVLowerSaddIntrinsics.h000066400000000000000000000056351506646577400241240ustar00rootroot00000000000000//===- SPIRVLowerSaddIntrinsics.h - sadd lowering --------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVLOWERSADDINTRINSICS_H #define SPIRV_SPIRVLOWERSADDINTRINSICS_H #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVLowerSaddIntrinsicsBase { public: SPIRVLowerSaddIntrinsicsBase() : Context(nullptr), Mod(nullptr) {} bool runLowerSaddIntrinsics(llvm::Module &M); private: void replaceSaddOverflow(llvm::Function &F); void replaceSaddSat(llvm::Function &F); llvm::LLVMContext *Context; llvm::Module *Mod; bool TheModuleIsModified = false; }; class SPIRVLowerSaddIntrinsicsPass : public llvm::PassInfoMixin, public SPIRVLowerSaddIntrinsicsBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM); }; class SPIRVLowerSaddIntrinsicsLegacy : public llvm::ModulePass, public SPIRVLowerSaddIntrinsicsBase { public: SPIRVLowerSaddIntrinsicsLegacy(); bool runOnModule(llvm::Module &M) override; static char ID; }; } // namespace SPIRV #endif // SPIRV_SPIRVLOWERSADDINTRINSICS_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVMDBuilder.h000066400000000000000000000106751506646577400220010ustar00rootroot00000000000000//===- SPIRVMDBuilder.h - SPIR-V metadata builder header file --*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file declares classes for creating SPIR-V metadata. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVMDBUILDER_H #define SPIRV_SPIRVMDBUILDER_H #include "SPIRVInternal.h" #include "llvm/IR/Metadata.h" #include using namespace llvm; namespace SPIRV { class SPIRVMDBuilder { public: template struct MDWrapper; struct NamedMDWrapper { NamedMDWrapper(NamedMDNode &Named, SPIRVMDBuilder &BB) : NMD(Named), B(BB) {} MDWrapper addOp() { return MDWrapper(*this, B); } NamedMDWrapper &addOp(MDWrapper &MD) { NMD.addOperand(MD.M); return *this; } NamedMDNode &NMD; SPIRVMDBuilder &B; }; template struct MDWrapper { MDWrapper(ParentT &Parent, SPIRVMDBuilder &Builder) : M(nullptr), P(Parent), B(Builder) {} MDWrapper &add(unsigned I) { V.push_back(ConstantAsMetadata::get(getUInt32(&B.M, I))); return *this; } MDWrapper &addU16(unsigned short I) { V.push_back(ConstantAsMetadata::get(getUInt16(&B.M, I))); return *this; } MDWrapper &add(StringRef S) { V.push_back(MDString::get(B.C, S)); return *this; } MDWrapper &add(Function *F) { V.push_back(ConstantAsMetadata::get(F)); return *this; } MDWrapper &add(SmallVectorImpl &S) { for (auto &I : S) add(I); return *this; } MDWrapper &addOp(MDNode *Node) { V.push_back(Node); return *this; } MDWrapper addOp() { return MDWrapper(*this, B); } MDWrapper &addOp(MDWrapper &MD) { V.push_back(MD.M); return *this; } /// Generate the scheduled MDNode and return the parent. /// If \param Ptr is not nullptr, save the generated MDNode. ParentT &done(MDNode **Ptr = nullptr) { M = MDNode::get(B.C, V); if (Ptr) *Ptr = M; return P.addOp(*this); } MDNode *M; ParentT &P; SPIRVMDBuilder &B; SmallVector V; }; explicit SPIRVMDBuilder(Module &Mod) : M(Mod), C(Mod.getContext()) {} NamedMDWrapper addNamedMD(StringRef Name) { return NamedMDWrapper(*M.getOrInsertNamedMetadata(Name), *this); } SPIRVMDBuilder &eraseNamedMD(StringRef Name) { if (auto N = M.getNamedMetadata(Name)) M.eraseNamedMetadata(N); return *this; } friend struct NamedMDWrapper; private: Module &M; LLVMContext &C; }; } /* namespace SPIRV */ #endif // SPIRV_SPIRVMDBUILDER_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVMDWalker.h000066400000000000000000000120651506646577400216330ustar00rootroot00000000000000//===- SPIRVMDWalker.h - SPIR-V metadata walker header file ----*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file declares classes for walking SPIR-V metadata. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVMDWALKER_H #define SPIRV_SPIRVMDWALKER_H #include "SPIRVInternal.h" #include "llvm/IR/Metadata.h" #include using namespace llvm; namespace SPIRV { class SPIRVMDWalker { public: template struct MDWrapper; struct NamedMDWrapper { NamedMDWrapper(NamedMDNode *Named, SPIRVMDWalker &WW) : NMD(Named), W(WW), I(0), Q(true) { E = Named ? Named->getNumOperands() : 0; } operator bool() const { return NMD; } bool atEnd() const { return !(NMD && I < E); } MDWrapper nextOp() { if (!Q) assert(I < E && "out of bound"); return MDWrapper( (NMD && I < E) ? NMD->getOperand(I++) : nullptr, *this, W); } NamedMDWrapper &setQuiet(bool Quiet) { Q = Quiet; return *this; } NamedMDNode *NMD; SPIRVMDWalker &W; unsigned I; unsigned E; bool Q; // Quiet }; template struct MDWrapper { MDWrapper(MDNode *Node, ParentT &Parent, SPIRVMDWalker &Walker) : M(Node), P(Parent), W(Walker), I(0), Q(false) { E = Node ? Node->getNumOperands() : 0; } operator bool() const { return M; } bool atEnd() const { return !(M && I < E); } template MDWrapper &get(T &V) { if (!Q) assert(I < E && "out of bound"); if (atEnd()) return *this; V = mdconst::dyn_extract(M->getOperand(I++))->getZExtValue(); return *this; } MDWrapper &get(std::string &S) { if (!Q) assert(I < E && "out of bound"); if (atEnd()) return *this; Metadata *Op = M->getOperand(I++); if (!Op) S = ""; else if (auto Str = dyn_cast(Op)) S = Str->getString().str(); else S = ""; return *this; } MDWrapper &get(Function *&F) { if (!Q) assert(I < E && "out of bound"); if (atEnd()) return *this; F = mdconst::dyn_extract(M->getOperand(I++)); return *this; } MDWrapper &get(SmallVectorImpl &SV) { if (atEnd()) return *this; while (I < E) { std::string S; get(S); SV.push_back(S); } return *this; } MDWrapper nextOp() { if (!Q) assert(I < E && "out of bound"); return MDWrapper( (M && I < E) ? dyn_cast(M->getOperand(I++)) : nullptr, *this, W); } ParentT &done() { return P; } MDWrapper &setQuiet(bool Quiet) { Q = Quiet; return *this; } MDNode *M; ParentT &P; SPIRVMDWalker &W; SmallVector V; unsigned I; unsigned E; bool Q; // Quiet }; explicit SPIRVMDWalker(Module &Mod) : M(Mod), C(Mod.getContext()) {} NamedMDWrapper getNamedMD(StringRef Name) { return NamedMDWrapper(M.getNamedMetadata(Name), *this); } friend struct NamedMDWrapper; private: Module &M; LLVMContext &C; }; } /* namespace SPIRV */ #endif // SPIRV_SPIRVMDWALKER_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVReader.cpp000066400000000000000000006350051506646577400217270ustar00rootroot00000000000000//===- SPIRVReader.cpp - Converts SPIR-V to LLVM ----------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements conversion of SPIR-V binary to LLVM IR. /// //===----------------------------------------------------------------------===// #include "SPIRVReader.h" #include "OCLUtil.h" #include "SPIRVAsm.h" #include "SPIRVBasicBlock.h" #include "SPIRVExtInst.h" #include "SPIRVFnVar.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVInternal.h" #include "SPIRVMDBuilder.h" #include "SPIRVMemAliasingINTEL.h" #include "SPIRVModule.h" #include "SPIRVToLLVMDbgTran.h" #include "SPIRVToOCL.h" #include "SPIRVType.h" #include "SPIRVUtil.h" #include "SPIRVValue.h" #include "VectorComputeUtil.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include #include #include #include #include #include #include #include #include #include #define DEBUG_TYPE "spirv" using namespace std; using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { cl::opt SPIRVEnableStepExpansion( "spirv-expand-step", cl::init(true), cl::desc("Enable expansion of OpenCL step and smoothstep function")); // Prefix for placeholder global variable name. const char *KPlaceholderPrefix = "placeholder."; // Save the translated LLVM before validation for debugging purpose. static bool DbgSaveTmpLLVM = false; static const char *DbgTmpLLVMFileName = "_tmp_llvmbil.ll"; namespace kOCLTypeQualifierName { const static char *Volatile = "volatile"; const static char *Restrict = "restrict"; const static char *Pipe = "pipe"; } // namespace kOCLTypeQualifierName static bool isKernel(SPIRVFunction *BF) { return BF->getModule()->isEntryPoint(ExecutionModelKernel, BF->getId()); } static void dumpLLVM(Module *M, const std::string &FName) { std::error_code EC; raw_fd_ostream FS(FName, EC, sys::fs::OF_None); if (!EC) { FS << *M; FS.close(); } } static MDNode *getMDNodeStringIntVec(LLVMContext *Context, const std::vector &IntVals) { std::vector ValueVec; for (auto &I : IntVals) ValueVec.push_back(ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), I))); return MDNode::get(*Context, ValueVec); } static MDNode *getMDTwoInt(LLVMContext *Context, unsigned Int1, unsigned Int2) { std::vector ValueVec; ValueVec.push_back(ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Int1))); ValueVec.push_back(ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Int2))); return MDNode::get(*Context, ValueVec); } static void addOCLVersionMetadata(LLVMContext *Context, Module *M, const std::string &MDName, unsigned Major, unsigned Minor) { NamedMDNode *NamedMD = M->getOrInsertNamedMetadata(MDName); NamedMD->addOperand(getMDTwoInt(Context, Major, Minor)); } static void addNamedMetadataStringSet(LLVMContext *Context, Module *M, const std::string &MDName, const std::set &StrSet) { NamedMDNode *NamedMD = M->getOrInsertNamedMetadata(MDName); std::vector ValueVec; for (auto &&Str : StrSet) { ValueVec.push_back(MDString::get(*Context, Str)); } NamedMD->addOperand(MDNode::get(*Context, ValueVec)); } static void addKernelArgumentMetadata( LLVMContext *Context, const std::string &MDName, SPIRVFunction *BF, llvm::Function *Fn, std::function ForeachFnArg) { std::vector ValueVec; BF->foreachArgument([&](SPIRVFunctionParameter *Arg) { ValueVec.push_back(ForeachFnArg(Arg)); }); Fn->setMetadata(MDName, MDNode::get(*Context, ValueVec)); } static void addBufferLocationMetadata( LLVMContext *Context, SPIRVFunction *BF, llvm::Function *Fn, std::function ForeachFnArg) { std::vector ValueVec; bool DecorationFound = false; BF->foreachArgument([&](SPIRVFunctionParameter *Arg) { if (Arg->getType()->isTypePointer() && Arg->hasDecorate(DecorationBufferLocationINTEL)) { DecorationFound = true; ValueVec.push_back(ForeachFnArg(Arg)); } else { llvm::Metadata *DefaultNode = ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), -1)); ValueVec.push_back(DefaultNode); } }); if (DecorationFound) Fn->setMetadata("kernel_arg_buffer_location", MDNode::get(*Context, ValueVec)); } static void addRuntimeAlignedMetadata( LLVMContext *Context, SPIRVFunction *BF, llvm::Function *Fn, std::function ForeachFnArg) { std::vector ValueVec; bool RuntimeAlignedFound = false; [[maybe_unused]] llvm::Metadata *DefaultNode = ConstantAsMetadata::get(ConstantInt::get(Type::getInt1Ty(*Context), 0)); BF->foreachArgument([&](SPIRVFunctionParameter *Arg) { if (Arg->hasAttr(FunctionParameterAttributeRuntimeAlignedINTEL) || Arg->hasDecorate(internal::DecorationRuntimeAlignedINTEL)) { RuntimeAlignedFound = true; ValueVec.push_back(ForeachFnArg(Arg)); } else { ValueVec.push_back(DefaultNode); } }); if (RuntimeAlignedFound) Fn->setMetadata("kernel_arg_runtime_aligned", MDNode::get(*Context, ValueVec)); } Value *SPIRVToLLVM::getTranslatedValue(SPIRVValue *BV) { auto Loc = ValueMap.find(BV); if (Loc != ValueMap.end()) return Loc->second; return nullptr; } static llvm::Optional translateSEVMetadata(SPIRVValue *BV, llvm::LLVMContext &Context) { llvm::Optional RetAttr; if (!BV->hasDecorate(DecorationSingleElementVectorINTEL)) return RetAttr; auto VecDecorateSEV = BV->getDecorations(DecorationSingleElementVectorINTEL); assert(VecDecorateSEV.size() == 1 && "Entry must have no more than one SingleElementVectorINTEL " "decoration"); auto *DecorateSEV = VecDecorateSEV.back(); auto LiteralCount = DecorateSEV->getLiteralCount(); assert(LiteralCount <= 1 && "SingleElementVectorINTEL decoration must " "have no more than one literal"); SPIRVWord IndirectLevelsOnElement = (LiteralCount == 1) ? DecorateSEV->getLiteral(0) : 0; RetAttr = Attribute::get(Context, kVCMetadata::VCSingleElementVector, std::to_string(IndirectLevelsOnElement)); return RetAttr; } IntrinsicInst *SPIRVToLLVM::getLifetimeStartIntrinsic(Instruction *I) { auto II = dyn_cast(I); if (II && II->getIntrinsicID() == Intrinsic::lifetime_start) return II; // Bitcast might be inserted during translation of OpLifetimeStart auto BC = dyn_cast(I); if (BC) { for (const auto &U : BC->users()) { II = dyn_cast(U); if (II && II->getIntrinsicID() == Intrinsic::lifetime_start) return II; ; } } return nullptr; } SPIRVErrorLog &SPIRVToLLVM::getErrorLog() { return BM->getErrorLog(); } void SPIRVToLLVM::setCallingConv(CallInst *Call) { Function *F = Call->getCalledFunction(); assert(F && "Function pointers are not allowed in SPIRV"); Call->setCallingConv(F->getCallingConv()); } // For integer types shorter than 32 bit, unsigned/signedness can be inferred // from zext/sext attribute. MDString *SPIRVToLLVM::transOCLKernelArgTypeName(SPIRVFunctionParameter *Arg) { auto Ty = Arg->isByVal() ? Arg->getType()->getPointerElementType() : Arg->getType(); return MDString::get(*Context, transTypeToOCLTypeName(Ty, !Arg->isZext())); } Value *SPIRVToLLVM::mapFunction(SPIRVFunction *BF, Function *F) { SPIRVDBG(spvdbgs() << "[mapFunction] " << *BF << " -> "; dbgs() << *F << '\n';) FuncMap[BF] = F; return F; } llvm::Optional SPIRVToLLVM::transIdAsConstant(SPIRVId Id) { auto *V = BM->get(Id); const auto *ConstValue = dyn_cast(transValue(V, nullptr, nullptr)); if (!ConstValue) return {}; return ConstValue->getZExtValue(); } llvm::Optional SPIRVToLLVM::getAlignment(SPIRVValue *V) { SPIRVWord AlignmentBytes = 0; if (V->hasAlignment(&AlignmentBytes)) { return AlignmentBytes; } // If there was no Alignment decoration, look for AlignmentId instead. SPIRVId AlignId; if (V->hasDecorateId(DecorationAlignmentId, 0, &AlignId)) { return transIdAsConstant(AlignId); } return {}; } Type *SPIRVToLLVM::transFPType(SPIRVType *T) { switch (T->getFloatBitWidth()) { case 16: if (T->isTypeFloat(16, FPEncodingBFloat16KHR)) return Type::getBFloatTy(*Context); return Type::getHalfTy(*Context); case 32: return Type::getFloatTy(*Context); case 64: return Type::getDoubleTy(*Context); default: llvm_unreachable("Invalid type"); return nullptr; } } std::string SPIRVToLLVM::transOCLImageTypeName(SPIRV::SPIRVTypeImage *ST) { return getSPIRVTypeName( kSPIRVTypeName::Image, getSPIRVImageTypePostfixes( getSPIRVImageSampledTypeName(ST->getSampledType()), ST->getDescriptor(), ST->hasAccessQualifier() ? ST->getAccessQualifier() : AccessQualifierReadOnly)); } std::string SPIRVToLLVM::transOCLSampledImageTypeName(SPIRV::SPIRVTypeSampledImage *ST) { return getSPIRVTypeName( kSPIRVTypeName::SampledImg, getSPIRVImageTypePostfixes( getSPIRVImageSampledTypeName(ST->getImageType()->getSampledType()), ST->getImageType()->getDescriptor(), ST->getImageType()->hasAccessQualifier() ? ST->getImageType()->getAccessQualifier() : AccessQualifierReadOnly)); } std::string SPIRVToLLVM::transVMEImageTypeName(SPIRV::SPIRVTypeVmeImageINTEL *VT) { return getSPIRVTypeName( kSPIRVTypeName::VmeImageINTEL, getSPIRVImageTypePostfixes( getSPIRVImageSampledTypeName(VT->getImageType()->getSampledType()), VT->getImageType()->getDescriptor(), VT->getImageType()->hasAccessQualifier() ? VT->getImageType()->getAccessQualifier() : AccessQualifierReadOnly)); } std::string SPIRVToLLVM::transPipeTypeName(SPIRV::SPIRVTypePipe *PT) { SPIRVAccessQualifierKind PipeAccess = PT->getAccessQualifier(); assert((PipeAccess == AccessQualifierReadOnly || PipeAccess == AccessQualifierWriteOnly) && "Invalid access qualifier"); return std::string(kSPIRVTypeName::PrefixAndDelim) + kSPIRVTypeName::Pipe + kSPIRVTypeName::Delimiter + kSPIRVTypeName::PostfixDelim + PipeAccess; } std::string SPIRVToLLVM::transOCLPipeStorageTypeName(SPIRV::SPIRVTypePipeStorage *PST) { return std::string(kSPIRVTypeName::PrefixAndDelim) + kSPIRVTypeName::PipeStorage; } std::string SPIRVToLLVM::transVCTypeName(SPIRVTypeBufferSurfaceINTEL *PST) { if (PST->hasAccessQualifier()) return VectorComputeUtil::getVCBufferSurfaceName(PST->getAccessQualifier()); return VectorComputeUtil::getVCBufferSurfaceName(); } Type *SPIRVToLLVM::transType(SPIRVType *T, bool IsClassMember) { auto Loc = TypeMap.find(T); if (Loc != TypeMap.end()) return Loc->second; SPIRVDBG(spvdbgs() << "[transType] " << *T << " -> ";) T->validate(); switch (static_cast(T->getOpCode())) { case OpTypeVoid: return mapType(T, Type::getVoidTy(*Context)); case OpTypeBool: return mapType(T, Type::getInt1Ty(*Context)); case OpTypeInt: return mapType(T, Type::getIntNTy(*Context, T->getIntegerBitWidth())); case OpTypeFloat: return mapType(T, transFPType(T)); case OpTypeArray: { // The length might be an OpSpecConstantOp, that needs to be specialized // and evaluated before the LLVM ArrayType can be constructed. auto *LenExpr = static_cast(T)->getLength(); auto *LenValue = cast(transValue(LenExpr, nullptr, nullptr)); return mapType(T, ArrayType::get(transType(T->getArrayElementType()), LenValue->getZExtValue())); } case internal::OpTypeTokenINTEL: return mapType(T, Type::getTokenTy(*Context)); case OpTypePointer: { unsigned AS = SPIRSPIRVAddrSpaceMap::rmap(T->getPointerStorageClass()); if (AS == SPIRAS_CodeSectionINTEL && !BM->shouldEmitFunctionPtrAddrSpace()) AS = SPIRAS_Private; if (BM->shouldEmitFunctionPtrAddrSpace() && T->getPointerElementType()->getOpCode() == OpTypeFunction) AS = SPIRAS_CodeSectionINTEL; return mapType( T, PointerType::get( transType(T->getPointerElementType(), IsClassMember), AS)); } case OpTypeVector: return mapType(T, FixedVectorType::get(transType(T->getVectorComponentType()), T->getVectorComponentCount())); case OpTypeMatrix: return mapType(T, ArrayType::get(transType(T->getMatrixColumnType()), T->getMatrixColumnCount())); case OpTypeOpaque: return mapType(T, StructType::create(*Context, T->getName())); case OpTypeFunction: { auto FT = static_cast(T); auto RT = transType(FT->getReturnType()); std::vector PT; for (size_t I = 0, E = FT->getNumParameters(); I != E; ++I) PT.push_back(transType(FT->getParameterType(I))); return mapType(T, FunctionType::get(RT, PT, false)); } case OpTypeImage: { auto ST = static_cast(T); if (ST->isOCLImage()) return mapType(T, getOrCreateOpaquePtrType(M, transOCLImageTypeName(ST))); else llvm_unreachable("Unsupported image type"); return nullptr; } case OpTypeSampledImage: { auto ST = static_cast(T); return mapType( T, getOrCreateOpaquePtrType(M, transOCLSampledImageTypeName(ST))); } case OpTypeStruct: { auto ST = static_cast(T); auto Name = ST->getName(); if (!Name.empty()) { if (auto OldST = StructType::getTypeByName(*Context, Name)) OldST->setName(""); } else { Name = "structtype"; } auto *StructTy = StructType::create(*Context, Name); mapType(ST, StructTy); SmallVector MT; for (size_t I = 0, E = ST->getMemberCount(); I != E; ++I) MT.push_back(transType(ST->getMemberType(I), true)); for (auto &CI : ST->getContinuedInstructions()) for (size_t I = 0, E = CI->getNumElements(); I != E; ++I) MT.push_back(transType(CI->getMemberType(I), true)); StructTy->setBody(MT, ST->isPacked()); return StructTy; } case OpTypePipe: { auto PT = static_cast(T); return mapType( T, getOrCreateOpaquePtrType(M, transPipeTypeName(PT), getOCLOpaqueTypeAddrSpace(T->getOpCode()))); } case OpTypePipeStorage: { auto PST = static_cast(T); return mapType( T, getOrCreateOpaquePtrType(M, transOCLPipeStorageTypeName(PST), getOCLOpaqueTypeAddrSpace(T->getOpCode()))); } case OpTypeVmeImageINTEL: { auto *VT = static_cast(T); return mapType(T, getOrCreateOpaquePtrType(M, transVMEImageTypeName(VT))); } case OpTypeBufferSurfaceINTEL: { auto PST = static_cast(T); return mapType(T, getOrCreateOpaquePtrType(M, transVCTypeName(PST), SPIRAddressSpace::SPIRAS_Global)); } case internal::OpTypeJointMatrixINTEL: { auto *MT = static_cast(T); auto R = static_cast(MT->getRows())->getZExtIntValue(); auto C = static_cast(MT->getColumns())->getZExtIntValue(); std::stringstream SS; SS << kSPIRVTypeName::PostfixDelim; SS << transTypeToOCLTypeName(MT->getCompType()); auto L = static_cast(MT->getLayout())->getZExtIntValue(); auto S = static_cast(MT->getScope())->getZExtIntValue(); SS << kSPIRVTypeName::PostfixDelim << R << kSPIRVTypeName::PostfixDelim << C << kSPIRVTypeName::PostfixDelim << L << kSPIRVTypeName::PostfixDelim << S; if (auto *Use = MT->getUse()) SS << kSPIRVTypeName::PostfixDelim << static_cast(Use)->getZExtIntValue(); std::string Name = getSPIRVTypeName(kSPIRVTypeName::JointMatrixINTEL, SS.str()); return mapType(T, getOrCreateOpaquePtrType(M, Name)); } case OpTypeCooperativeMatrixKHR: { auto *MT = static_cast(T); unsigned Scope = static_cast(MT->getScope())->getZExtIntValue(); unsigned Rows = static_cast(MT->getRows())->getZExtIntValue(); unsigned Cols = static_cast(MT->getColumns())->getZExtIntValue(); unsigned Use = static_cast(MT->getUse())->getZExtIntValue(); std::stringstream SS; SS << kSPIRVTypeName::PostfixDelim; SS << transTypeToOCLTypeName(MT->getCompType()); SS << kSPIRVTypeName::PostfixDelim << Scope << kSPIRVTypeName::PostfixDelim << Rows << kSPIRVTypeName::PostfixDelim << Cols << kSPIRVTypeName::PostfixDelim << Use; std::string Name = getSPIRVTypeName(kSPIRVTypeName::CooperativeMatrixKHR, SS.str()); return mapType(T, getOrCreateOpaquePtrType(M, Name)); } case OpTypeForwardPointer: { SPIRVTypeForwardPointer *FP = static_cast(static_cast(T)); return mapType(T, transType(static_cast( BM->getEntry(FP->getPointerId())))); } default: { auto OC = T->getOpCode(); if (isOpaqueGenericTypeOpCode(OC) || isSubgroupAvcINTELTypeOpCode(OC)) return mapType(T, getSPIRVOpaquePtrType(M, OC)); llvm_unreachable("Not implemented!"); } } return 0; } std::string SPIRVToLLVM::transTypeToOCLTypeName(SPIRVType *T, bool IsSigned) { switch (T->getOpCode()) { case OpTypeVoid: return "void"; case OpTypeBool: return "bool"; case OpTypeInt: { std::string Prefix = IsSigned ? "" : "u"; switch (T->getIntegerBitWidth()) { case 8: return Prefix + "char"; case 16: return Prefix + "short"; case 32: return Prefix + "int"; case 64: return Prefix + "long"; default: // Arbitrary precision integer return Prefix + std::string("int") + T->getIntegerBitWidth() + "_t"; } } break; case OpTypeFloat: switch (T->getFloatBitWidth()) { case 16: if (static_cast(T)->getFloatingPointEncoding() == FPEncodingBFloat16KHR) return "bfloat16"; return "half"; case 32: return "float"; case 64: return "double"; default: llvm_unreachable("invalid floating pointer bitwidth"); return std::string("float") + T->getFloatBitWidth() + "_t"; } break; case OpTypeArray: return "array"; case OpTypePointer: { SPIRVType *ET = T->getPointerElementType(); if (isa(ET)) { SPIRVTypeFunction *TF = static_cast(ET); std::string name = transTypeToOCLTypeName(TF->getReturnType()); name += " (*)("; for (unsigned I = 0, E = TF->getNumParameters(); I < E; ++I) name += transTypeToOCLTypeName(TF->getParameterType(I)) + ','; name.back() = ')'; // replace the last comma with a closing brace. return name; } return transTypeToOCLTypeName(ET) + "*"; } case OpTypeVector: return transTypeToOCLTypeName(T->getVectorComponentType()) + T->getVectorComponentCount(); case OpTypeMatrix: return transTypeToOCLTypeName(T->getMatrixColumnType()) + T->getMatrixColumnCount(); case OpTypeOpaque: return T->getName(); case OpTypeFunction: llvm_unreachable("Unsupported"); return "function"; case OpTypeStruct: { auto Name = T->getName(); if (Name.find("struct.") == 0) Name[6] = ' '; else if (Name.find("union.") == 0) Name[5] = ' '; return Name; } case OpTypePipe: return "pipe"; case OpTypeSampler: return "sampler_t"; case OpTypeImage: { std::string Name; Name = rmap(static_cast(T)->getDescriptor()); return Name; } default: if (isOpaqueGenericTypeOpCode(T->getOpCode())) { return OCLOpaqueTypeOpCodeMap::rmap(T->getOpCode()); } llvm_unreachable("Not implemented"); return "unknown"; } } std::vector SPIRVToLLVM::getPointerElementTypes(ArrayRef Tys) { std::vector PointerElementTys; for (SPIRVType *T : Tys) { PointerIndirectPair PtrElemTy; switch (static_cast(T->getOpCode())) { case OpTypePointer: { SPIRVType *UntransTy = T->getPointerElementType(); PtrElemTy.setPointer(transType(UntransTy)); if (PtrElemTy.getPointer()->isPointerTy()) { PtrElemTy = getPointerElementTypes(UntransTy)[0]; PtrElemTy.setInt(true); } break; } case OpTypeImage: { auto *ST = static_cast(T); if (ST->isOCLImage()) PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transOCLImageTypeName(ST))); break; } case OpTypeSampledImage: { auto *ST = static_cast(T); PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transOCLSampledImageTypeName(ST))); break; } case OpTypePipe: { auto *PT = static_cast(T); PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transPipeTypeName(PT))); break; } case OpTypePipeStorage: { auto *PST = static_cast(T); PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transOCLPipeStorageTypeName(PST))); break; } case OpTypeVmeImageINTEL: { auto *VT = static_cast(T); PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transVMEImageTypeName(VT))); break; } case OpTypeBufferSurfaceINTEL: { auto *PST = static_cast(T); PtrElemTy.setPointer( getOrCreateOpaqueStructType(M, transVCTypeName(PST))); break; } case internal::OpTypeJointMatrixINTEL: { auto *MT = static_cast(T); auto R = static_cast(MT->getRows())->getZExtIntValue(); auto C = static_cast(MT->getColumns())->getZExtIntValue(); std::stringstream SS; SS << kSPIRVTypeName::PostfixDelim; SS << transTypeToOCLTypeName(MT->getCompType()); auto L = static_cast(MT->getLayout())->getZExtIntValue(); auto S = static_cast(MT->getScope())->getZExtIntValue(); SS << kSPIRVTypeName::PostfixDelim << R << kSPIRVTypeName::PostfixDelim << C << kSPIRVTypeName::PostfixDelim << L << kSPIRVTypeName::PostfixDelim << S; if (auto *Use = MT->getUse()) SS << kSPIRVTypeName::PostfixDelim << static_cast(Use)->getZExtIntValue(); std::string Name = getSPIRVTypeName(kSPIRVTypeName::JointMatrixINTEL, SS.str()); PtrElemTy.setPointer(getOrCreateOpaqueStructType(M, Name)); break; } case OpTypeCooperativeMatrixKHR: { auto *MT = static_cast(T); auto R = static_cast(MT->getRows())->getZExtIntValue(); auto C = static_cast(MT->getColumns())->getZExtIntValue(); std::stringstream SS; SS << kSPIRVTypeName::PostfixDelim; SS << transTypeToOCLTypeName(MT->getCompType()); auto S = static_cast(MT->getScope())->getZExtIntValue(); SS << kSPIRVTypeName::PostfixDelim << S << kSPIRVTypeName::PostfixDelim << R << kSPIRVTypeName::PostfixDelim << C; if (auto *Use = MT->getUse()) SS << kSPIRVTypeName::PostfixDelim << static_cast(Use)->getZExtIntValue(); std::string Name = getSPIRVTypeName(kSPIRVTypeName::CooperativeMatrixKHR, SS.str()); PtrElemTy.setPointer(getOrCreateOpaqueStructType(M, Name)); break; } case OpTypeFunction: { // A function parameter will get converted into a function pointer later, // so make sure that the pointer element type gets the actual function // type. PtrElemTy.setPointer(transType(T)); break; } default: { auto OC = T->getOpCode(); if (isOpaqueGenericTypeOpCode(OC) || isSubgroupAvcINTELTypeOpCode(OC)) PtrElemTy.setPointer(getOrCreateOpaqueStructType( M, getSPIRVTypeName(SPIRVOpaqueTypeOpCodeMap::rmap(OC)))); } } PointerElementTys.push_back(PtrElemTy); } return PointerElementTys; } std::vector SPIRVToLLVM::transTypeVector(const std::vector &BT) { std::vector T; for (auto I : BT) T.push_back(transType(I)); return T; } std::vector SPIRVToLLVM::transValue(const std::vector &BV, Function *F, BasicBlock *BB) { std::vector V; for (auto I : BV) V.push_back(transValue(I, F, BB)); return V; } void SPIRVToLLVM::setName(llvm::Value *V, SPIRVValue *BV) { auto Name = BV->getName(); if (!Name.empty() && (!V->hasName() || Name != V->getName())) V->setName(Name); } inline llvm::Metadata *SPIRVToLLVM::getMetadataFromName(std::string Name) { return llvm::MDNode::get(*Context, llvm::MDString::get(*Context, Name)); } inline std::vector SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name, SPIRVWord Parameter) { return {MDString::get(*Context, Name), ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Parameter))}; } inline llvm::MDNode * SPIRVToLLVM::getMetadataFromNameAndParameter(std::string Name, int64_t Parameter) { std::vector Metadata = { MDString::get(*Context, Name), ConstantAsMetadata::get( ConstantInt::get(Type::getInt64Ty(*Context), Parameter))}; return llvm::MDNode::get(*Context, Metadata); } template void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM, const Loop *LoopObj) { if (!LM) return; auto Temp = MDNode::getTemporary(*Context, None); auto Self = MDNode::get(*Context, Temp.get()); Self->replaceOperandWith(0, Self); SPIRVWord LC = LM->getLoopControl(); if (LC == LoopControlMaskNone) { LoopObj->setLoopID(Self); return; } unsigned NumParam = 0; std::vector Metadata; std::vector LoopControlParameters = LM->getLoopControlParameters(); Metadata.push_back(llvm::MDNode::get(*Context, Self)); // To correctly decode loop control parameters, order of checks for loop // control masks must match with the order given in the spec (see 3.23), // i.e. check smaller-numbered bits first. // Unroll and UnrollCount loop controls can't be applied simultaneously with // DontUnroll loop control. if (LC & LoopControlUnrollMask && !(LC & LoopControlPartialCountMask)) Metadata.push_back(getMetadataFromName("llvm.loop.unroll.enable")); else if (LC & LoopControlDontUnrollMask) Metadata.push_back(getMetadataFromName("llvm.loop.unroll.disable")); if (LC & LoopControlDependencyInfiniteMask) Metadata.push_back(getMetadataFromName("llvm.loop.ivdep.enable")); if (LC & LoopControlDependencyLengthMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.ivdep.safelen", LoopControlParameters[NumParam]))); ++NumParam; // TODO: Fix the increment/assertion logic in all of the conditions assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } // Placeholder for LoopControls added in SPIR-V 1.4 spec (see 3.23) if (LC & LoopControlMinIterationsMask) { ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlMaxIterationsMask) { ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlIterationMultipleMask) { ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlPeelCountMask) { ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlPartialCountMask && !(LC & LoopControlDontUnrollMask)) { // If unroll factor is set as '1' and Unroll mask is applied attempt to do // full unrolling and disable it if the trip count is not known at compile // time. if (1 == LoopControlParameters[NumParam] && (LC & LoopControlUnrollMask)) Metadata.push_back(getMetadataFromName("llvm.loop.unroll.full")); else Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.unroll.count", LoopControlParameters[NumParam]))); ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlInitiationIntervalINTELMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter( "llvm.loop.ii.count", LoopControlParameters[NumParam]))); ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlMaxConcurrencyINTELMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.max_concurrency.count", LoopControlParameters[NumParam]))); ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlDependencyArrayINTELMask) { // Collect pointer variable <-> safelen information std::map PointerSflnMap; unsigned NumOperandPairs = LoopControlParameters[NumParam]; unsigned OperandsEndIndex = NumParam + NumOperandPairs * 2; assert(OperandsEndIndex <= LoopControlParameters.size() && "Missing loop control parameter!"); SPIRVModule *M = LM->getModule(); while (NumParam < OperandsEndIndex) { SPIRVId ArraySPIRVId = LoopControlParameters[++NumParam]; Value *PointerVar = ValueMap[M->getValue(ArraySPIRVId)]; unsigned Safelen = LoopControlParameters[++NumParam]; PointerSflnMap.emplace(PointerVar, Safelen); } // A single run over the loop to retrieve all GetElementPtr instructions // that access relevant array variables std::map> ArrayGEPMap; for (const auto &BB : LoopObj->blocks()) { for (Instruction &I : *BB) { auto *GEP = dyn_cast(&I); if (!GEP) continue; Value *AccessedPointer = GEP->getPointerOperand(); if (auto *LI = dyn_cast(AccessedPointer)) AccessedPointer = LI->getPointerOperand(); auto PointerSflnIt = PointerSflnMap.find(AccessedPointer); if (PointerSflnIt != PointerSflnMap.end()) { ArrayGEPMap[AccessedPointer].push_back(GEP); } } } // Create index group metadata nodes - one per each of the array // variables. Mark each GEP accessing a particular array variable // into a corresponding index group std::map> SafelenIdxGroupMap; // Whenever a kernel closure field access is pointed to instead of // an array/pointer variable, ensure that all GEPs to that memory // share the same index group by hashing the newly added index groups. // "Memory offset info" represents a handle to the whole closure block // + an integer offset to a particular captured parameter. using MemoryOffsetInfo = std::pair; std::map OffsetIdxGroupMap; for (auto &ArrayGEPIt : ArrayGEPMap) { MDNode *CurrentDepthIdxGroup = nullptr; if (auto *PrecedingGEP = dyn_cast(ArrayGEPIt.first)) { Value *ClosureFieldPointer = PrecedingGEP->getPointerOperand(); unsigned Offset = cast(PrecedingGEP->getOperand(2))->getZExtValue(); MemoryOffsetInfo Info{ClosureFieldPointer, Offset}; auto OffsetIdxGroupIt = OffsetIdxGroupMap.find(Info); if (OffsetIdxGroupIt == OffsetIdxGroupMap.end()) { // This is the first GEP encountered for this closure field. // Emit a distinct index group that will be referenced from // llvm.loop.parallel_access_indices metadata; hash the new // MDNode for future accesses to the same memory. CurrentDepthIdxGroup = llvm::MDNode::getDistinct(*Context, None); OffsetIdxGroupMap.emplace(Info, CurrentDepthIdxGroup); } else { // Previous accesses to that field have already been indexed, // just use the already-existing metadata. CurrentDepthIdxGroup = OffsetIdxGroupIt->second; } } else /* Regular kernel-scope array/pointer variable */ { // Emit a distinct index group that will be referenced from // llvm.loop.parallel_access_indices metadata CurrentDepthIdxGroup = llvm::MDNode::getDistinct(*Context, None); } unsigned Safelen = PointerSflnMap.find(ArrayGEPIt.first)->second; SafelenIdxGroupMap[Safelen].insert(CurrentDepthIdxGroup); for (auto *GEP : ArrayGEPIt.second) { StringRef IdxGroupMDName("llvm.index.group"); llvm::MDNode *PreviousIdxGroup = GEP->getMetadata(IdxGroupMDName); if (!PreviousIdxGroup) { GEP->setMetadata(IdxGroupMDName, CurrentDepthIdxGroup); continue; } // If we're dealing with an embedded loop, it may be the case // that GEP instructions for some of the arrays were already // marked by the algorithm when it went over the outer level loops. // In order to retain the IVDep information for each "loop // dimension", we will mark such GEP's into a separate joined node // that will refer to the previous levels' index groups AND to the // index group specific to the current loop. std::vector CurrentDepthOperands( PreviousIdxGroup->op_begin(), PreviousIdxGroup->op_end()); if (CurrentDepthOperands.empty()) CurrentDepthOperands.push_back(PreviousIdxGroup); CurrentDepthOperands.push_back(CurrentDepthIdxGroup); auto *JointIdxGroup = llvm::MDNode::get(*Context, CurrentDepthOperands); GEP->setMetadata(IdxGroupMDName, JointIdxGroup); } } for (auto &SflnIdxGroupIt : SafelenIdxGroupMap) { auto *Name = MDString::get(*Context, "llvm.loop.parallel_access_indices"); unsigned SflnValue = SflnIdxGroupIt.first; llvm::Metadata *SafelenMDOp = SflnValue ? ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(*Context), SflnValue)) : nullptr; std::vector Parameters{Name}; for (auto *Node : SflnIdxGroupIt.second) Parameters.push_back(Node); if (SafelenMDOp) Parameters.push_back(SafelenMDOp); Metadata.push_back(llvm::MDNode::get(*Context, Parameters)); } ++NumParam; } if (LC & LoopControlPipelineEnableINTELMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.intel.pipelining.enable", LoopControlParameters[NumParam++]))); assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlLoopCoalesceINTELMask) { // If LoopCoalesce has a parameter of '0' if (!LoopControlParameters[NumParam]) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromName("llvm.loop.coalesce.enable"))); } else { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.coalesce.count", LoopControlParameters[NumParam]))); } ++NumParam; assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlMaxInterleavingINTELMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter("llvm.loop.max_interleaving.count", LoopControlParameters[NumParam++]))); assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlSpeculatedIterationsINTELMask) { Metadata.push_back(llvm::MDNode::get( *Context, getMetadataFromNameAndParameter( "llvm.loop.intel.speculated.iterations.count", LoopControlParameters[NumParam++]))); assert(NumParam <= LoopControlParameters.size() && "Missing loop control parameter!"); } if (LC & LoopControlNoFusionINTELMask) Metadata.push_back(getMetadataFromName("llvm.loop.fusion.disable")); if (LC & spv::internal::LoopControlLoopCountINTELMask) { // LoopCountINTELMask parameters are int64 and each parameter is stored // as 2 SPIRVWords (int32) assert(NumParam + 6 <= LoopControlParameters.size() && "Missing loop control parameter!"); uint64_t LoopCountMin = static_cast(LoopControlParameters[NumParam++]); LoopCountMin |= static_cast(LoopControlParameters[NumParam++]) << 32; if (static_cast(LoopCountMin) >= 0) { Metadata.push_back(getMetadataFromNameAndParameter( "llvm.loop.intel.loopcount_min", static_cast(LoopCountMin))); } uint64_t LoopCountMax = static_cast(LoopControlParameters[NumParam++]); LoopCountMax |= static_cast(LoopControlParameters[NumParam++]) << 32; if (static_cast(LoopCountMax) >= 0) { Metadata.push_back(getMetadataFromNameAndParameter( "llvm.loop.intel.loopcount_max", static_cast(LoopCountMax))); } uint64_t LoopCountAvg = static_cast(LoopControlParameters[NumParam++]); LoopCountAvg |= static_cast(LoopControlParameters[NumParam++]) << 32; if (static_cast(LoopCountAvg) >= 0) { Metadata.push_back(getMetadataFromNameAndParameter( "llvm.loop.intel.loopcount_avg", static_cast(LoopCountAvg))); } } llvm::MDNode *Node = llvm::MDNode::get(*Context, Metadata); // Set the first operand to refer itself Node->replaceOperandWith(0, Node); LoopObj->setLoopID(Node); } void SPIRVToLLVM::transLLVMLoopMetadata(const Function *F) { assert(F); if (FuncLoopMetadataMap.empty()) return; // Function declaration doesn't contain loop metadata. if (F->isDeclaration()) return; DominatorTree DomTree(*(const_cast(F))); LoopInfo LI(DomTree); // In SPIRV loop metadata is linked to a header basic block of a loop // whilst in LLVM IR it is linked to a latch basic block (the one // whose back edge goes to a header basic block) of the loop. // To ensure consistent behaviour, we can rely on the `llvm::Loop` // class to handle the metadata placement for (const auto *LoopObj : LI.getLoopsInPreorder()) { // Check that loop header BB contains loop metadata. const auto LMDItr = FuncLoopMetadataMap.find(LoopObj->getHeader()); if (LMDItr == FuncLoopMetadataMap.end()) continue; const auto *LMD = LMDItr->second; if (LMD->getOpCode() == OpLoopMerge) { const auto *LM = static_cast(LMD); setLLVMLoopMetadata(LM, LoopObj); } else if (LMD->getOpCode() == OpLoopControlINTEL) { const auto *LCI = static_cast(LMD); setLLVMLoopMetadata(LCI, LoopObj); } FuncLoopMetadataMap.erase(LMDItr); } } Value *SPIRVToLLVM::transValue(SPIRVValue *BV, Function *F, BasicBlock *BB, bool CreatePlaceHolder) { SPIRVToLLVMValueMap::iterator Loc = ValueMap.find(BV); if (Loc != ValueMap.end() && (!PlaceholderMap.count(BV) || CreatePlaceHolder)) return Loc->second; SPIRVDBG(spvdbgs() << "[transValue] " << *BV << " -> ";) BV->validate(); auto V = transValueWithoutDecoration(BV, F, BB, CreatePlaceHolder); if (!V) { SPIRVDBG(dbgs() << " Warning ! nullptr\n";) return nullptr; } setName(V, BV); if (!transDecoration(BV, V)) { assert(0 && "trans decoration fail"); return nullptr; } SPIRVDBG(dbgs() << *V << '\n';) return V; } Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F, BasicBlock *BB) { SPIRVUnary *BC = static_cast(BV); auto Src = transValue(BC->getOperand(0), F, BB, BB ? true : false); auto Dst = transType(BC->getType()); CastInst::CastOps CO = Instruction::BitCast; bool IsExt = Dst->getScalarSizeInBits() > Src->getType()->getScalarSizeInBits(); switch (BC->getOpCode()) { case OpPtrCastToGeneric: case OpGenericCastToPtr: case OpPtrCastToCrossWorkgroupINTEL: case OpCrossWorkgroupCastToPtrINTEL: { // If module has pointers with DeviceOnlyINTEL and HostOnlyINTEL storage // classes there will be a situation, when global_device/global_host // address space will be lowered to just global address space. If there also // is an addrspacecast - we need to replace it with source pointer. if (Src->getType()->getPointerAddressSpace() == Dst->getPointerAddressSpace()) return Src; CO = Instruction::AddrSpaceCast; break; } case OpSConvert: CO = IsExt ? Instruction::SExt : Instruction::Trunc; break; case OpUConvert: CO = IsExt ? Instruction::ZExt : Instruction::Trunc; break; case OpFConvert: CO = IsExt ? Instruction::FPExt : Instruction::FPTrunc; break; case OpBitcast: // OpBitcast need to be handled as a special-case when the source is a // pointer and the destination is not a pointer, and where the source is not // a pointer and the destination is a pointer. This is supported by the // SPIR-V bitcast, but not by the LLVM bitcast. CO = Instruction::BitCast; if (Src->getType()->isPointerTy() && !Dst->isPointerTy()) { if (auto *DstVecTy = dyn_cast(Dst)) { unsigned TotalBitWidth = DstVecTy->getElementType()->getIntegerBitWidth() * DstVecTy->getNumElements(); auto *IntTy = Type::getIntNTy(BB->getContext(), TotalBitWidth); if (BB) { Src = CastInst::CreatePointerCast(Src, IntTy, "", BB); } else { Src = ConstantExpr::getPointerCast(dyn_cast(Src), IntTy); } } else { CO = Instruction::PtrToInt; } } else if (!Src->getType()->isPointerTy() && Dst->isPointerTy()) { if (auto *SrcVecTy = dyn_cast(Src->getType())) { unsigned TotalBitWidth = SrcVecTy->getElementType()->getIntegerBitWidth() * SrcVecTy->getNumElements(); auto *IntTy = Type::getIntNTy(BB->getContext(), TotalBitWidth); if (BB) { Src = CastInst::Create(Instruction::BitCast, Src, IntTy, "", BB); } else { Src = ConstantExpr::getBitCast(dyn_cast(Src), IntTy); } } CO = Instruction::IntToPtr; } break; default: CO = static_cast(OpCodeMap::rmap(BC->getOpCode())); } assert(CastInst::isCast(CO) && "Invalid cast op code"); SPIRVDBG(if (!CastInst::castIsValid(CO, Src, Dst)) { spvdbgs() << "Invalid cast: " << *BV << " -> "; dbgs() << "Op = " << CO << ", Src = " << *Src << " Dst = " << *Dst << '\n'; }) if (BB) return CastInst::Create(CO, Src, Dst, BV->getName(), BB); return ConstantExpr::getCast(CO, dyn_cast(Src), Dst); } static void applyNoIntegerWrapDecorations(const SPIRVValue *BV, Instruction *Inst) { if (BV->hasDecorate(DecorationNoSignedWrap)) { Inst->setHasNoSignedWrap(true); } if (BV->hasDecorate(DecorationNoUnsignedWrap)) { Inst->setHasNoUnsignedWrap(true); } } static void applyFPFastMathModeDecorations(const SPIRVValue *BV, Instruction *Inst) { SPIRVWord V; FastMathFlags FMF; if (BV->hasDecorate(DecorationFPFastMathMode, 0, &V)) { if (V & FPFastMathModeNotNaNMask) FMF.setNoNaNs(); if (V & FPFastMathModeNotInfMask) FMF.setNoInfs(); if (V & FPFastMathModeNSZMask) FMF.setNoSignedZeros(); if (V & FPFastMathModeAllowRecipMask) FMF.setAllowReciprocal(); if (V & FPFastMathModeAllowContractFastINTELMask) FMF.setAllowContract(); if (V & FPFastMathModeAllowReassocINTELMask) FMF.setAllowReassoc(); if (V & FPFastMathModeFastMask) FMF.setFast(); Inst->setFastMathFlags(FMF); } } Value *SPIRVToLLVM::transShiftLogicalBitwiseInst(SPIRVValue *BV, BasicBlock *BB, Function *F) { SPIRVBinary *BBN = static_cast(BV); if (BV->getType()->isTypeCooperativeMatrixKHR()) { return mapValue(BV, transSPIRVBuiltinFromInst(BBN, BB)); } Instruction::BinaryOps BO; auto OP = BBN->getOpCode(); if (isLogicalOpCode(OP)) OP = IntBoolOpMap::rmap(OP); BO = static_cast(OpCodeMap::rmap(OP)); Value *Op0 = transValue(BBN->getOperand(0), F, BB); Value *Op1 = transValue(BBN->getOperand(1), F, BB); IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } Value *NewOp = Builder.CreateBinOp(BO, Op0, Op1, BV->getName()); if (auto *Inst = dyn_cast(NewOp)) { applyNoIntegerWrapDecorations(BV, Inst); applyFPFastMathModeDecorations(BV, Inst); } return NewOp; } Value *SPIRVToLLVM::transCmpInst(SPIRVValue *BV, BasicBlock *BB, Function *F) { SPIRVCompare *BC = static_cast(BV); SPIRVType *BT = BC->getOperand(0)->getType(); Value *Inst = nullptr; auto OP = BC->getOpCode(); if (isLogicalOpCode(OP)) OP = IntBoolOpMap::rmap(OP); Value *Op0 = transValue(BC->getOperand(0), F, BB); Value *Op1 = transValue(BC->getOperand(1), F, BB); IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } if (OP == OpLessOrGreater) OP = OpFOrdNotEqual; if (BT->isTypeVectorOrScalarInt() || BT->isTypeVectorOrScalarBool() || BT->isTypePointer()) Inst = Builder.CreateICmp(CmpMap::rmap(OP), Op0, Op1); else if (BT->isTypeVectorOrScalarFloat()) Inst = Builder.CreateFCmp(CmpMap::rmap(OP), Op0, Op1); assert(Inst && "not implemented"); applyFPFastMathModeDecorations(BV, static_cast(Inst)); return Inst; } Type *SPIRVToLLVM::mapType(SPIRVType *BT, Type *T) { SPIRVDBG(dbgs() << *T << '\n';) TypeMap[BT] = T; return T; } Value *SPIRVToLLVM::mapValue(SPIRVValue *BV, Value *V) { auto Loc = ValueMap.find(BV); if (Loc != ValueMap.end()) { if (Loc->second == V) return V; auto LD = dyn_cast(Loc->second); auto Placeholder = dyn_cast(LD->getPointerOperand()); assert(LD && Placeholder && Placeholder->getName().startswith(KPlaceholderPrefix) && "A value is translated twice"); // Replaces placeholders for PHI nodes LD->replaceAllUsesWith(V); LD->eraseFromParent(); Placeholder->eraseFromParent(); } ValueMap[BV] = V; return V; } CallInst * SPIRVToLLVM::expandOCLBuiltinWithScalarArg(CallInst *CI, const std::string &FuncName) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); if (!CI->getOperand(0)->getType()->isVectorTy() && CI->getOperand(1)->getType()->isVectorTy()) { return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { auto VecElemCount = cast(CI->getOperand(1)->getType())->getElementCount(); Value *NewVec = nullptr; if (auto CA = dyn_cast(Args[0])) NewVec = ConstantVector::getSplat(VecElemCount, CA); else { NewVec = ConstantVector::getSplat( VecElemCount, Constant::getNullValue(Args[0]->getType())); NewVec = InsertElementInst::Create(NewVec, Args[0], getInt32(M, 0), "", CI); NewVec = new ShuffleVectorInst( NewVec, NewVec, ConstantVector::getSplat(VecElemCount, getInt32(M, 0)), "", CI); } NewVec->takeName(Args[0]); Args[0] = NewVec; return FuncName; }, &Attrs); } return CI; } std::string SPIRVToLLVM::transOCLPipeTypeAccessQualifier(SPIRV::SPIRVTypePipe *ST) { return SPIRSPIRVAccessQualifierMap::rmap(ST->getAccessQualifier()); } void SPIRVToLLVM::transGeneratorMD() { SPIRVMDBuilder B(*M); B.addNamedMD(kSPIRVMD::Generator) .addOp() .addU16(BM->getGeneratorId()) .addU16(BM->getGeneratorVer()) .done(); } Value *SPIRVToLLVM::oclTransConstantSampler(SPIRV::SPIRVConstantSampler *BCS, BasicBlock *BB) { auto *SamplerT = getSPIRVOpaquePtrType(M, OpTypeSampler); auto *I32Ty = IntegerType::getInt32Ty(*Context); auto *FTy = FunctionType::get(SamplerT, {I32Ty}, false); FunctionCallee Func = M->getOrInsertFunction(SAMPLER_INIT, FTy); auto Lit = (BCS->getAddrMode() << 1) | BCS->getNormalized() | ((BCS->getFilterMode() + 1) << 4); return CallInst::Create(Func, {ConstantInt::get(I32Ty, Lit)}, "", BB); } Value *SPIRVToLLVM::oclTransConstantPipeStorage( SPIRV::SPIRVConstantPipeStorage *BCPS) { string CPSName = string(kSPIRVTypeName::PrefixAndDelim) + kSPIRVTypeName::ConstantPipeStorage; auto Int32Ty = IntegerType::getInt32Ty(*Context); auto CPSTy = StructType::getTypeByName(*Context, CPSName); if (!CPSTy) { Type *CPSElemsTy[] = {Int32Ty, Int32Ty, Int32Ty}; CPSTy = StructType::create(*Context, CPSElemsTy, CPSName); } assert(CPSTy != nullptr && "Could not create spirv.ConstantPipeStorage"); Constant *CPSElems[] = {ConstantInt::get(Int32Ty, BCPS->getPacketSize()), ConstantInt::get(Int32Ty, BCPS->getPacketAlign()), ConstantInt::get(Int32Ty, BCPS->getCapacity())}; return new GlobalVariable(*M, CPSTy, false, GlobalValue::LinkOnceODRLinkage, ConstantStruct::get(CPSTy, CPSElems), BCPS->getName(), nullptr, GlobalValue::NotThreadLocal, SPIRAS_Global); } // A pointer annotation may have been generated for the operand. If the operand // is used further in IR, it should be replaced with the intrinsic call result. // Otherwise, the generated pointer annotation call is left unused. static void replaceOperandWithAnnotationIntrinsicCallResult(Value *&V) { if (Use *SingleUse = V->getSingleUndroppableUse()) { if (auto *II = dyn_cast(SingleUse->getUser())) { if (II->getIntrinsicID() == Intrinsic::ptr_annotation && II->getType() == V->getType()) // Overwrite the future operand with the intrinsic call result. V = II; } } } // Translate aliasing memory access masks for SPIRVLoad and SPIRVStore // instructions. These masks are mapped on alias.scope and noalias // metadata in LLVM. Translation of optional string operand isn't yet supported // in the translator. template void SPIRVToLLVM::transAliasingMemAccess(SPIRVInstType *BI, Instruction *I) { static_assert(std::is_same::value || std::is_same::value, "Only stores and loads can be aliased by memory access mask"); if (BI->SPIRVMemoryAccess::isNoAlias()) addMemAliasMetadata(I, BI->SPIRVMemoryAccess::getNoAliasInstID(), LLVMContext::MD_noalias); if (BI->SPIRVMemoryAccess::isAliasScope()) addMemAliasMetadata(I, BI->SPIRVMemoryAccess::getAliasScopeInstID(), LLVMContext::MD_alias_scope); } // Create and apply alias.scope/noalias metadata void SPIRVToLLVM::addMemAliasMetadata(Instruction *I, SPIRVId AliasListId, uint32_t AliasMDKind) { SPIRVAliasScopeListDeclINTEL *AliasList = BM->get(AliasListId); std::vector AliasScopeIds = AliasList->getArguments(); MDBuilder MDB(*Context); SmallVector MDScopes; for (const auto ScopeId : AliasScopeIds) { SPIRVAliasScopeDeclINTEL *AliasScope = BM->get(ScopeId); std::vector AliasDomainIds = AliasScope->getArguments(); // Currently we expect exactly one argument for aliasing scope // instruction. // TODO: add translation of string scope and domain operand. assert(AliasDomainIds.size() == 1 && "AliasScopeDeclINTEL must have exactly one argument"); SPIRVId AliasDomainId = AliasDomainIds[0]; // Create and store unique domain and scope metadata MDAliasDomainMap.emplace(AliasDomainId, MDB.createAnonymousAliasScopeDomain()); MDAliasScopeMap.emplace(ScopeId, MDB.createAnonymousAliasScope( MDAliasDomainMap[AliasDomainId])); MDScopes.emplace_back(MDAliasScopeMap[ScopeId]); } // Create and store unique alias.scope/noalias metadata MDAliasListMap.emplace(AliasListId, MDNode::concatenate(I->getMetadata(AliasMDKind), MDNode::get(*Context, MDScopes))); I->setMetadata(AliasMDKind, MDAliasListMap[AliasListId]); } void SPIRVToLLVM::transFunctionPointerCallArgumentAttributes( SPIRVValue *BV, CallInst *CI, SPIRVTypeFunction *CalledFnTy) { std::vector ArgumentAttributes = BV->getDecorations(internal::DecorationArgumentAttributeINTEL); for (const auto *Dec : ArgumentAttributes) { std::vector Literals = Dec->getVecLiteral(); SPIRVWord ArgNo = Literals[0]; SPIRVWord SpirvAttr = Literals[1]; Attribute::AttrKind LlvmAttrKind = SPIRSPIRVFuncParamAttrMap::rmap( static_cast(SpirvAttr)); auto LlvmAttr = Attribute::isTypeAttrKind(LlvmAttrKind) ? Attribute::get(CI->getContext(), LlvmAttrKind, transType(CalledFnTy->getParameterType(ArgNo) ->getPointerElementType())) : Attribute::get(CI->getContext(), LlvmAttrKind); CI->addParamAttr(ArgNo, LlvmAttr); } } /// For instructions, this function assumes they are created in order /// and appended to the given basic block. An instruction may use a /// instruction from another BB which has not been translated. Such /// instructions should be translated to place holders at the point /// of first use, then replaced by real instructions when they are /// created. /// /// When CreatePlaceHolder is true, create a load instruction of a /// global variable as placeholder for SPIRV instruction. Otherwise, /// create instruction and replace placeholder if there is one. Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, BasicBlock *BB, bool CreatePlaceHolder) { auto OC = BV->getOpCode(); IntBoolOpMap::rfind(OC, &OC); // Translation of non-instruction values switch (OC) { case OpConstant: case OpSpecConstant: { SPIRVConstant *BConst = static_cast(BV); SPIRVType *BT = BV->getType(); Type *LT = transType(BT); uint64_t ConstValue = BConst->getZExtIntValue(); SPIRVWord SpecId = 0; if (OC == OpSpecConstant && BV->hasDecorate(DecorationSpecId, 0, &SpecId)) { // Update the value with possibly provided external specialization. if (BM->getSpecializationConstant(SpecId, ConstValue)) { assert( (BT->getBitWidth() == 64 || (ConstValue >> BT->getBitWidth()) == 0) && "Size of externally provided specialization constant value doesn't" "fit into the specialization constant type"); } } switch (BT->getOpCode()) { case OpTypeBool: case OpTypeInt: { const unsigned NumBits = BT->getBitWidth(); if (NumBits > 64) { // Translate huge arbitrary precision integer constants const unsigned RawDataNumWords = BConst->getNumWords(); const unsigned BigValNumWords = (RawDataNumWords + 1) / 2; std::vector BigValVec(BigValNumWords); const std::vector &RawData = BConst->getSPIRVWords(); // SPIRV words are integers of 32-bit width, meanwhile llvm::APInt // is storing data using an array of 64-bit words. Here we pack SPIRV // words into 64-bit integer array. for (size_t I = 0; I != RawDataNumWords / 2; ++I) BigValVec[I] = (static_cast(RawData[2 * I + 1]) << SpirvWordBitWidth) | RawData[2 * I]; if (RawDataNumWords % 2) BigValVec.back() = RawData.back(); return mapValue(BV, ConstantInt::get(LT, APInt(NumBits, BigValVec))); } return mapValue( BV, ConstantInt::get(LT, ConstValue, static_cast(BT)->isSigned())); } case OpTypeFloat: { const llvm::fltSemantics *FS = nullptr; switch (BT->getFloatBitWidth()) { case 16: FS = (BT->isTypeFloat(16, FPEncodingBFloat16KHR) ? &APFloat::BFloat() : &APFloat::IEEEhalf()); break; case 32: FS = &APFloat::IEEEsingle(); break; case 64: FS = &APFloat::IEEEdouble(); break; default: llvm_unreachable("invalid floating-point type"); } APFloat FPConstValue(*FS, APInt(BT->getFloatBitWidth(), ConstValue)); return mapValue(BV, ConstantFP::get(*Context, FPConstValue)); } default: llvm_unreachable("Not implemented"); return nullptr; } } case OpConstantTrue: return mapValue(BV, ConstantInt::getTrue(*Context)); case OpConstantFalse: return mapValue(BV, ConstantInt::getFalse(*Context)); case OpSpecConstantTrue: case OpSpecConstantFalse: { bool IsTrue = OC == OpSpecConstantTrue; SPIRVWord SpecId = 0; if (BV->hasDecorate(DecorationSpecId, 0, &SpecId)) { uint64_t ConstValue = 0; if (BM->getSpecializationConstant(SpecId, ConstValue)) { IsTrue = ConstValue; } } return mapValue(BV, IsTrue ? ConstantInt::getTrue(*Context) : ConstantInt::getFalse(*Context)); } case OpConstantNull: { auto LT = transType(BV->getType()); return mapValue(BV, Constant::getNullValue(LT)); } case OpConstantComposite: case OpSpecConstantComposite: { auto BCC = static_cast(BV); std::vector CV; for (auto &I : BCC->getElements()) CV.push_back(dyn_cast(transValue(I, F, BB))); for (auto &CI : BCC->getContinuedInstructions()) { for (auto &I : CI->getElements()) CV.push_back(dyn_cast(transValue(I, F, BB))); } switch (BV->getType()->getOpCode()) { case OpTypeVector: return mapValue(BV, ConstantVector::get(CV)); case OpTypeMatrix: case OpTypeArray: { auto *AT = cast(transType(BCC->getType())); for (size_t I = 0; I != AT->getNumElements(); ++I) { auto *ElemTy = AT->getElementType(); if (auto *ElemPtrTy = dyn_cast(ElemTy)) { assert(isa(CV[I]->getType()) && "Constant type doesn't match constexpr array element type"); if (ElemPtrTy->getAddressSpace() != cast(CV[I]->getType())->getAddressSpace()) CV[I] = ConstantExpr::getAddrSpaceCast(CV[I], AT->getElementType()); } } return mapValue(BV, ConstantArray::get(AT, CV)); } case OpTypeStruct: { auto BCCTy = dyn_cast(transType(BCC->getType())); auto Members = BCCTy->getNumElements(); auto Constants = CV.size(); // if we try to initialize constant TypeStruct, add bitcasts // if src and dst types are both pointers but to different types if (Members == Constants) { for (unsigned I = 0; I < Members; ++I) { if (CV[I]->getType() == BCCTy->getElementType(I)) continue; if (!CV[I]->getType()->isPointerTy() || !BCCTy->getElementType(I)->isPointerTy()) continue; if (cast(CV[I]->getType())->getAddressSpace() != cast(BCCTy->getElementType(I))->getAddressSpace()) CV[I] = ConstantExpr::getAddrSpaceCast(CV[I], BCCTy->getElementType(I)); else CV[I] = ConstantExpr::getBitCast(CV[I], BCCTy->getElementType(I)); } } return mapValue(BV, ConstantStruct::get( dyn_cast(transType(BCC->getType())), CV)); } case OpTypeCooperativeMatrixKHR: { assert(CV.size() == 1 && "expecting exactly one operand for cooperative matrix types"); llvm::Type *RetTy = transType(BCC->getType()); llvm::Type *EltTy = transType( static_cast(BV->getType()) ->getCompType()); auto *FTy = FunctionType::get(RetTy, {EltTy}, false); FunctionCallee Func = M->getOrInsertFunction(getSPIRVFuncName(OC, RetTy), FTy); IRBuilder<> Builder(BB); CallInst *Call = Builder.CreateCall(Func, CV.front()); Call->setCallingConv(CallingConv::SPIR_FUNC); return Call; } default: llvm_unreachable("not implemented"); return nullptr; } } case OpConstantSampler: { auto BCS = static_cast(BV); // Intentially do not map this value. We want to generate constant // sampler initializer every time constant sampler is used, otherwise // initializer may not dominate all its uses. return oclTransConstantSampler(BCS, BB); } case OpConstantPipeStorage: { auto BCPS = static_cast(BV); return mapValue(BV, oclTransConstantPipeStorage(BCPS)); } case OpSpecConstantOp: { auto BI = createInstFromSpecConstantOp(static_cast(BV)); return mapValue(BV, transValue(BI, nullptr, nullptr, false)); } case OpConstantFunctionPointerINTEL: { SPIRVConstantFunctionPointerINTEL *BC = static_cast(BV); SPIRVFunction *F = BC->getFunction(); BV->setName(F->getName()); const unsigned AS = BM->shouldEmitFunctionPtrAddrSpace() ? SPIRAS_CodeSectionINTEL : SPIRAS_Private; return mapValue(BV, transFunction(F, AS)); } case OpUndef: return mapValue(BV, UndefValue::get(transType(BV->getType()))); case OpSizeOf: { Type *ResTy = transType(BV->getType()); auto *BI = static_cast(BV); SPIRVType *TypeArg = reinterpret_cast(BI->getOpValue(0)); Type *EltTy = transType(TypeArg->getPointerElementType()); uint64_t Size = M->getDataLayout().getTypeStoreSize(EltTy).getFixedValue(); return mapValue(BV, ConstantInt::get(ResTy, Size)); } case OpVariable: { auto BVar = static_cast(BV); auto *PreTransTy = BVar->getType()->getPointerElementType(); auto *Ty = transType(PreTransTy); bool IsConst = BVar->isConstant(); llvm::GlobalValue::LinkageTypes LinkageTy = transLinkageType(BVar); SPIRVStorageClassKind BS = BVar->getStorageClass(); SPIRVValue *Init = BVar->getInitializer(); if (PreTransTy->isTypeSampler() && BS == StorageClassUniformConstant) { // Skip generating llvm code during translation of a variable definition, // generate code only for its uses if (!BB) return nullptr; assert(Init && "UniformConstant OpVariable with sampler type must have " "an initializer!"); return transValue(Init, F, BB); } if (BS == StorageClassFunction && !Init) { assert(BB && "Invalid BB"); return mapValue(BV, new AllocaInst(Ty, 0, BV->getName(), BB)); } SPIRAddressSpace AddrSpace; bool IsVectorCompute = BVar->hasDecorate(DecorationVectorComputeVariableINTEL); Constant *Initializer = nullptr; if (IsVectorCompute) { AddrSpace = VectorComputeUtil::getVCGlobalVarAddressSpace(BS); Initializer = UndefValue::get(Ty); } else AddrSpace = SPIRSPIRVAddrSpaceMap::rmap(BS); // Force SPIRV BuiltIn variable's name to be __spirv_BuiltInXXXX. // No matter what BV's linkage name is. SPIRVBuiltinVariableKind BVKind; if (BVar->isBuiltin(&BVKind)) BV->setName(prefixSPIRVName(SPIRVBuiltInNameMap::map(BVKind))); auto LVar = new GlobalVariable(*M, Ty, IsConst, LinkageTy, /*Initializer=*/nullptr, BV->getName(), 0, GlobalVariable::NotThreadLocal, AddrSpace); auto Res = mapValue(BV, LVar); if (Init) Initializer = dyn_cast(transValue(Init, F, BB, false)); else if (LinkageTy == GlobalValue::CommonLinkage) // In LLVM, variables with common linkage type must be initialized to 0. Initializer = Constant::getNullValue(Ty); else if (BS == SPIRVStorageClassKind::StorageClassWorkgroup && LinkageTy != GlobalValue::ExternalLinkage) Initializer = dyn_cast(UndefValue::get(Ty)); else if ((LinkageTy != GlobalValue::ExternalLinkage) && (BS == SPIRVStorageClassKind::StorageClassCrossWorkgroup)) Initializer = Constant::getNullValue(Ty); LVar->setUnnamedAddr((IsConst && Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy(8)) ? GlobalValue::UnnamedAddr::Global : GlobalValue::UnnamedAddr::None); LVar->setInitializer(Initializer); if (IsVectorCompute) { LVar->addAttribute(kVCMetadata::VCGlobalVariable); SPIRVWord Offset; if (BVar->hasDecorate(DecorationGlobalVariableOffsetINTEL, 0, &Offset)) LVar->addAttribute(kVCMetadata::VCByteOffset, utostr(Offset)); if (BVar->hasDecorate(DecorationVolatile)) LVar->addAttribute(kVCMetadata::VCVolatile); auto SEVAttr = translateSEVMetadata(BVar, LVar->getContext()); if (SEVAttr) LVar->addAttribute(SEVAttr.getValue().getKindAsString(), SEVAttr.getValue().getValueAsString()); } return Res; } case OpFunctionParameter: { auto BA = static_cast(BV); assert(F && "Invalid function"); unsigned ArgNo = 0; for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I, ++ArgNo) { if (ArgNo == BA->getArgNo()) return mapValue(BV, &(*I)); } llvm_unreachable("Invalid argument"); return nullptr; } case OpFunction: return mapValue(BV, transFunction(static_cast(BV))); case OpAsmINTEL: return mapValue(BV, transAsmINTEL(static_cast(BV))); case OpLabel: return mapValue(BV, BasicBlock::Create(*Context, BV->getName(), F)); case OpSpecConstantArchitectureINTEL: llvm_unreachable( "Encountered non-specialized OpSpecConstantArchitectureINTEL"); return nullptr; case OpSpecConstantTargetINTEL: llvm_unreachable("Encountered non-specialized OpSpecConstantTargetINTEL"); return nullptr; case OpSpecConstantCapabilitiesINTEL: llvm_unreachable( "Encountered non-specialized OpSpecConstantCapabilitiesINTEL"); return nullptr; default: // do nothing break; } // During translation of OpSpecConstantOp we create an instruction // corresponding to the Opcode operand and then translate this instruction. // For such instruction BB and F should be nullptr, because it is a constant // expression declared out of scope of any basic block or function. // All other values require valid BB pointer. assert(((isSpecConstantOpAllowedOp(OC) && !F && !BB) || BB) && "Invalid BB"); // Creation of place holder if (CreatePlaceHolder) { auto *Ty = transType(BV->getType()); auto GV = new GlobalVariable(*M, Ty, false, GlobalValue::PrivateLinkage, nullptr, std::string(KPlaceholderPrefix) + BV->getName(), 0, GlobalVariable::NotThreadLocal, 0); auto LD = new LoadInst(Ty, GV, BV->getName(), BB); PlaceholderMap[BV] = LD; return mapValue(BV, LD); } // Translation of instructions int OpCode = BV->getOpCode(); switch (OpCode) { case OpVariableLengthArrayINTEL: { auto *VLA = static_cast(BV); llvm::Type *Ty = transType(BV->getType()->getPointerElementType()); llvm::Value *ArrSize = transValue(VLA->getOperand(0), F, BB); return mapValue( BV, new AllocaInst(Ty, SPIRAS_Private, ArrSize, BV->getName(), BB)); } case OpRestoreMemoryINTEL: { auto *Restore = static_cast(BV); llvm::Value *Ptr = transValue(Restore->getOperand(0), F, BB); Function *StackRestore = Intrinsic::getDeclaration(M, Intrinsic::stackrestore); return mapValue(BV, CallInst::Create(StackRestore, {Ptr}, "", BB)); } case OpSaveMemoryINTEL: { Function *StackSave = Intrinsic::getDeclaration(M, Intrinsic::stacksave); return mapValue(BV, CallInst::Create(StackSave, "", BB)); } case OpBranch: { auto *BR = static_cast(BV); auto *BI = BranchInst::Create( cast(transValue(BR->getTargetLabel(), F, BB)), BB); // Loop metadata will be translated in the end of function translation. return mapValue(BV, BI); } case OpBranchConditional: { auto *BR = static_cast(BV); auto *BC = BranchInst::Create( cast(transValue(BR->getTrueLabel(), F, BB)), cast(transValue(BR->getFalseLabel(), F, BB)), transValue(BR->getCondition(), F, BB), BB); // Loop metadata will be translated in the end of function translation. return mapValue(BV, BC); } case OpPhi: { auto Phi = static_cast(BV); auto LPhi = dyn_cast(mapValue( BV, PHINode::Create(transType(Phi->getType()), Phi->getPairs().size() / 2, Phi->getName(), BB))); Phi->foreachPair([&](SPIRVValue *IncomingV, SPIRVBasicBlock *IncomingBB, size_t Index) { auto Translated = transValue(IncomingV, F, BB); LPhi->addIncoming(Translated, dyn_cast(transValue(IncomingBB, F, BB))); }); return LPhi; } case OpUnreachable: return mapValue(BV, new UnreachableInst(*Context, BB)); case OpReturn: return mapValue(BV, ReturnInst::Create(*Context, BB)); case OpReturnValue: { auto RV = static_cast(BV); return mapValue( BV, ReturnInst::Create(*Context, transValue(RV->getReturnValue(), F, BB), BB)); } case OpLifetimeStart: { SPIRVLifetimeStart *LTStart = static_cast(BV); IRBuilder<> Builder(BB); SPIRVWord Size = LTStart->getSize(); ConstantInt *S = nullptr; if (Size) S = Builder.getInt64(Size); Value *Var = transValue(LTStart->getObject(), F, BB); CallInst *Start = Builder.CreateLifetimeStart(Var, S); return mapValue(BV, Start); } case OpLifetimeStop: { SPIRVLifetimeStop *LTStop = static_cast(BV); IRBuilder<> Builder(BB); SPIRVWord Size = LTStop->getSize(); ConstantInt *S = nullptr; if (Size) S = Builder.getInt64(Size); auto Var = transValue(LTStop->getObject(), F, BB); for (const auto &I : Var->users()) if (auto II = getLifetimeStartIntrinsic(dyn_cast(I))) return mapValue(BV, Builder.CreateLifetimeEnd(II->getOperand(1), S)); return mapValue(BV, Builder.CreateLifetimeEnd(Var, S)); } case OpStore: { SPIRVStore *BS = static_cast(BV); StoreInst *SI = nullptr; auto *Src = transValue(BS->getSrc(), F, BB); auto *Dst = transValue(BS->getDst(), F, BB); // A ptr.annotation may have been generated for the source variable. replaceOperandWithAnnotationIntrinsicCallResult(Src); // A ptr.annotation may have been generated for the destination variable. replaceOperandWithAnnotationIntrinsicCallResult(Dst); bool isVolatile = BS->SPIRVMemoryAccess::isVolatile(); uint64_t AlignValue = BS->SPIRVMemoryAccess::getAlignment(); if (0 == AlignValue) SI = new StoreInst(Src, Dst, isVolatile, BB); else SI = new StoreInst(Src, Dst, isVolatile, Align(AlignValue), BB); if (BS->SPIRVMemoryAccess::isNonTemporal()) transNonTemporalMetadata(SI); transAliasingMemAccess(BS, SI); return mapValue(BV, SI); } case OpLoad: { SPIRVLoad *BL = static_cast(BV); auto *V = transValue(BL->getSrc(), F, BB); // A ptr.annotation may have been generated for the source variable. replaceOperandWithAnnotationIntrinsicCallResult(V); Type *Ty = transType(BL->getType()); LoadInst *LI = nullptr; uint64_t AlignValue = BL->SPIRVMemoryAccess::getAlignment(); if (0 == AlignValue) { LI = new LoadInst(Ty, V, BV->getName(), BL->SPIRVMemoryAccess::isVolatile(), BB); } else { LI = new LoadInst(Ty, V, BV->getName(), BL->SPIRVMemoryAccess::isVolatile(), Align(AlignValue), BB); } if (BL->SPIRVMemoryAccess::isNonTemporal()) transNonTemporalMetadata(LI); transAliasingMemAccess(BL, LI); return mapValue(BV, LI); } case OpCopyMemory: { auto *BC = static_cast(BV); llvm::Value *Dst = transValue(BC->getTarget(), F, BB); MaybeAlign Align(BC->getAlignment()); MaybeAlign SrcAlign = BC->getSrcAlignment() ? MaybeAlign(BC->getSrcAlignment()) : Align; Type *EltTy = transType(BC->getSource()->getType()->getPointerElementType()); uint64_t Size = M->getDataLayout().getTypeStoreSize(EltTy).getFixedValue(); bool IsVolatile = BC->SPIRVMemoryAccess::isVolatile(); IRBuilder<> Builder(BB); llvm::Value *Src = transValue(BC->getSource(), F, BB); CallInst *CI = Builder.CreateMemCpy(Dst, Align, Src, SrcAlign, Size, IsVolatile); if (isFuncNoUnwind()) CI->getFunction()->addFnAttr(Attribute::NoUnwind); return mapValue(BV, CI); } case OpCopyMemorySized: { SPIRVCopyMemorySized *BC = static_cast(BV); llvm::Value *Dst = transValue(BC->getTarget(), F, BB); MaybeAlign Align(BC->getAlignment()); MaybeAlign SrcAlign = BC->getSrcAlignment() ? MaybeAlign(BC->getSrcAlignment()) : Align; llvm::Value *Size = transValue(BC->getSize(), F, BB); bool IsVolatile = BC->SPIRVMemoryAccess::isVolatile(); IRBuilder<> Builder(BB); llvm::Value *Src = transValue(BC->getSource(), F, BB); CallInst *CI = Builder.CreateMemCpy(Dst, Align, Src, SrcAlign, Size, IsVolatile); if (isFuncNoUnwind()) CI->getFunction()->addFnAttr(Attribute::NoUnwind); return mapValue(BV, CI); } case OpSelect: { SPIRVSelect *BS = static_cast(BV); IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } return mapValue(BV, Builder.CreateSelect(transValue(BS->getCondition(), F, BB), transValue(BS->getTrueValue(), F, BB), transValue(BS->getFalseValue(), F, BB), BV->getName())); } case OpLine: case OpSelectionMerge: // OpenCL Compiler does not use this instruction return nullptr; case OpLoopMerge: // Will be translated after all other function's case OpLoopControlINTEL: // instructions are translated. FuncLoopMetadataMap[BB] = BV; return nullptr; case OpSwitch: { auto BS = static_cast(BV); auto Select = transValue(BS->getSelect(), F, BB); auto LS = SwitchInst::Create( Select, dyn_cast(transValue(BS->getDefault(), F, BB)), BS->getNumPairs(), BB); BS->foreachPair( [&](SPIRVSwitch::LiteralTy Literals, SPIRVBasicBlock *Label) { assert(!Literals.empty() && "Literals should not be empty"); assert(Literals.size() <= 2 && "Number of literals should not be more then two"); uint64_t Literal = uint64_t(Literals.at(0)); if (Literals.size() == 2) { Literal += uint64_t(Literals.at(1)) << 32; } LS->addCase( ConstantInt::get(cast(Select->getType()), Literal), cast(transValue(Label, F, BB))); }); return mapValue(BV, LS); } case OpVectorTimesScalar: { auto VTS = static_cast(BV); IRBuilder<> Builder(BB); auto Scalar = transValue(VTS->getScalar(), F, BB); auto Vector = transValue(VTS->getVector(), F, BB); auto *VecTy = cast(Vector->getType()); unsigned VecSize = VecTy->getNumElements(); auto NewVec = Builder.CreateVectorSplat(VecSize, Scalar, Scalar->getName()); NewVec->takeName(Scalar); auto Scale = Builder.CreateFMul(Vector, NewVec, "scale"); return mapValue(BV, Scale); } case OpVectorTimesMatrix: { auto *VTM = static_cast(BV); IRBuilder<> Builder(BB); Value *Mat = transValue(VTM->getMatrix(), F, BB); Value *Vec = transValue(VTM->getVector(), F, BB); // Vec is of N elements. // Mat is of M columns and N rows. // Mat consists of vectors: V_1, V_2, ..., V_M // // The product is: // // |------- M ----------| // Result = sum ( {Vec_1, Vec_1, ..., Vec_1} * {V_1_1, V_2_1, ..., V_M_1}, // {Vec_2, Vec_2, ..., Vec_2} * {V_1_2, V_2_2, ..., V_M_2}, // ... // {Vec_N, Vec_N, ..., Vec_N} * {V_1_N, V_2_N, ..., V_M_N}); unsigned M = Mat->getType()->getArrayNumElements(); auto *VecTy = cast(Vec->getType()); FixedVectorType *VTy = FixedVectorType::get(VecTy->getElementType(), M); auto ETy = VTy->getElementType(); unsigned N = VecTy->getNumElements(); Value *V = Builder.CreateVectorSplat(M, ConstantFP::get(ETy, 0.0)); for (unsigned Idx = 0; Idx != N; ++Idx) { Value *S = Builder.CreateExtractElement(Vec, Builder.getInt32(Idx)); Value *Lhs = Builder.CreateVectorSplat(M, S); Value *Rhs = UndefValue::get(VTy); for (unsigned Idx2 = 0; Idx2 != M; ++Idx2) { Value *Vx = Builder.CreateExtractValue(Mat, Idx2); Value *Vxi = Builder.CreateExtractElement(Vx, Builder.getInt32(Idx)); Rhs = Builder.CreateInsertElement(Rhs, Vxi, Builder.getInt32(Idx2)); } Value *Mul = Builder.CreateFMul(Lhs, Rhs); V = Builder.CreateFAdd(V, Mul); } return mapValue(BV, V); } case OpMatrixTimesScalar: { auto MTS = static_cast(BV); IRBuilder<> Builder(BB); auto Scalar = transValue(MTS->getScalar(), F, BB); auto Matrix = transValue(MTS->getMatrix(), F, BB); uint64_t ColNum = Matrix->getType()->getArrayNumElements(); auto ColType = cast(Matrix->getType())->getElementType(); auto VecSize = cast(ColType)->getNumElements(); auto NewVec = Builder.CreateVectorSplat(VecSize, Scalar, Scalar->getName()); NewVec->takeName(Scalar); Value *V = UndefValue::get(Matrix->getType()); for (uint64_t Idx = 0; Idx != ColNum; Idx++) { auto Col = Builder.CreateExtractValue(Matrix, Idx); auto I = Builder.CreateFMul(Col, NewVec); V = Builder.CreateInsertValue(V, I, Idx); } return mapValue(BV, V); } case OpMatrixTimesVector: { auto *MTV = static_cast(BV); IRBuilder<> Builder(BB); Value *Mat = transValue(MTV->getMatrix(), F, BB); Value *Vec = transValue(MTV->getVector(), F, BB); // Result is similar to Matrix * Matrix // Mat is of M columns and N rows. // Mat consists of vectors: V_1, V_2, ..., V_M // where each vector is of size N. // // Vec is of size M. // The product is a vector of size N. // // |------- N ----------| // Result = sum ( {Vec_1, Vec_1, ..., Vec_1} * V_1, // {Vec_2, Vec_2, ..., Vec_2} * V_2, // ... // {Vec_M, Vec_M, ..., Vec_M} * V_N ); // // where sum is defined as vector sum. unsigned M = Mat->getType()->getArrayNumElements(); FixedVectorType *VTy = cast( cast(Mat->getType())->getElementType()); unsigned N = VTy->getNumElements(); auto ETy = VTy->getElementType(); Value *V = Builder.CreateVectorSplat(N, ConstantFP::get(ETy, 0.0)); for (unsigned Idx = 0; Idx != M; ++Idx) { Value *S = Builder.CreateExtractElement(Vec, Builder.getInt32(Idx)); Value *Lhs = Builder.CreateVectorSplat(N, S); Value *Vx = Builder.CreateExtractValue(Mat, Idx); Value *Mul = Builder.CreateFMul(Lhs, Vx); V = Builder.CreateFAdd(V, Mul); } return mapValue(BV, V); } case OpMatrixTimesMatrix: { auto *MTM = static_cast(BV); IRBuilder<> Builder(BB); Value *M1 = transValue(MTM->getLeftMatrix(), F, BB); Value *M2 = transValue(MTM->getRightMatrix(), F, BB); // Each matrix consists of a list of columns. // M1 (the left matrix) is of C1 columns and R1 rows. // M1 consists of a list of vectors: V_1, V_2, ..., V_C1 // where V_x are vectors of size R1. // // M2 (the right matrix) is of C2 columns and R2 rows. // M2 consists of a list of vectors: U_1, U_2, ..., U_C2 // where U_x are vectors of size R2. // // Now M1 * M2 requires C1 == R2. // The result is a matrix of C2 columns and R1 rows. // That is, consists of C2 vectors of size R1. // // M1 * M2 algorithm is as below: // // Result = { dot_product(U_1, M1), // dot_product(U_2, M1), // ... // dot_product(U_C2, M1) }; // where // dot_product (U, M) is defined as: // // |-------- C1 ------| // Result = sum ( {U[1], U[1], ..., U[1]} * V_1, // {U[2], U[2], ..., U[2]} * V_2, // ... // {U[R2], U[R2], ..., U[R2]} * V_C1 ); // Note that C1 == R2 // sum is defined as vector sum. unsigned C1 = M1->getType()->getArrayNumElements(); unsigned C2 = M2->getType()->getArrayNumElements(); FixedVectorType *V1Ty = cast(cast(M1->getType())->getElementType()); FixedVectorType *V2Ty = cast(cast(M2->getType())->getElementType()); unsigned R1 = V1Ty->getNumElements(); unsigned R2 = V2Ty->getNumElements(); auto ETy = V1Ty->getElementType(); (void)C1; assert(C1 == R2 && "Unmatched matrix"); auto VTy = FixedVectorType::get(ETy, R1); auto ResultTy = ArrayType::get(VTy, C2); Value *Res = UndefValue::get(ResultTy); for (unsigned Idx = 0; Idx != C2; ++Idx) { Value *U = Builder.CreateExtractValue(M2, Idx); // Calculate dot_product(U, M1) Value *Dot = Builder.CreateVectorSplat(R1, ConstantFP::get(ETy, 0.0)); for (unsigned Idx2 = 0; Idx2 != R2; ++Idx2) { Value *Ux = Builder.CreateExtractElement(U, Builder.getInt32(Idx2)); Value *Lhs = Builder.CreateVectorSplat(R1, Ux); Value *Rhs = Builder.CreateExtractValue(M1, Idx2); Value *Mul = Builder.CreateFMul(Lhs, Rhs); Dot = Builder.CreateFAdd(Dot, Mul); } Res = Builder.CreateInsertValue(Res, Dot, Idx); } return mapValue(BV, Res); } case OpTranspose: { auto TR = static_cast(BV); IRBuilder<> Builder(BB); auto Matrix = transValue(TR->getMatrix(), F, BB); unsigned ColNum = Matrix->getType()->getArrayNumElements(); FixedVectorType *ColTy = cast( cast(Matrix->getType())->getElementType()); unsigned RowNum = ColTy->getNumElements(); auto VTy = FixedVectorType::get(ColTy->getElementType(), ColNum); auto ResultTy = ArrayType::get(VTy, RowNum); Value *V = UndefValue::get(ResultTy); SmallVector MCache; MCache.reserve(ColNum); for (unsigned Idx = 0; Idx != ColNum; ++Idx) MCache.push_back(Builder.CreateExtractValue(Matrix, Idx)); if (ColNum == RowNum) { // Fastpath switch (ColNum) { case 2: { Value *V1 = Builder.CreateShuffleVector(MCache[0], MCache[1], ArrayRef{0, 2}); V = Builder.CreateInsertValue(V, V1, 0); Value *V2 = Builder.CreateShuffleVector(MCache[0], MCache[1], ArrayRef{1, 3}); V = Builder.CreateInsertValue(V, V2, 1); return mapValue(BV, V); } case 4: { for (int Idx = 0; Idx < 4; ++Idx) { Value *V1 = Builder.CreateShuffleVector(MCache[0], MCache[1], ArrayRef{Idx, Idx + 4}); Value *V2 = Builder.CreateShuffleVector(MCache[2], MCache[3], ArrayRef{Idx, Idx + 4}); Value *V3 = Builder.CreateShuffleVector(V1, V2, ArrayRef{0, 1, 2, 3}); V = Builder.CreateInsertValue(V, V3, Idx); } return mapValue(BV, V); } default: break; } } // Slowpath for (unsigned Idx = 0; Idx != RowNum; ++Idx) { Value *Vec = UndefValue::get(VTy); for (unsigned Idx2 = 0; Idx2 != ColNum; ++Idx2) { Value *S = Builder.CreateExtractElement(MCache[Idx2], Builder.getInt32(Idx)); Vec = Builder.CreateInsertElement(Vec, S, Idx2); } V = Builder.CreateInsertValue(V, Vec, Idx); } return mapValue(BV, V); } case OpCopyObject: { SPIRVCopyObject *CO = static_cast(BV); auto *Ty = transType(CO->getOperand()->getType()); AllocaInst *AI = new AllocaInst(Ty, 0, "", BB); new StoreInst(transValue(CO->getOperand(), F, BB), AI, BB); LoadInst *LI = new LoadInst(Ty, AI, "", BB); return mapValue(BV, LI); } case OpCopyLogical: { SPIRVCopyLogical *CL = static_cast(BV); auto *SrcTy = transType(CL->getOperand()->getType()); auto *DstTy = transType(CL->getType()); assert(M->getDataLayout().getTypeStoreSize(SrcTy).getFixedValue() == M->getDataLayout().getTypeStoreSize(DstTy).getFixedValue() && "Size mismatch in OpCopyLogical"); IRBuilder<> Builder(BB); auto *SrcAI = Builder.CreateAlloca(SrcTy); Builder.CreateAlignedStore(transValue(CL->getOperand(), F, BB), SrcAI, SrcAI->getAlign()); auto *LI = Builder.CreateAlignedLoad(DstTy, SrcAI, SrcAI->getAlign()); return mapValue(BV, LI); } case OpAccessChain: case OpInBoundsAccessChain: case OpPtrAccessChain: case OpInBoundsPtrAccessChain: { auto AC = static_cast(BV); auto Base = transValue(AC->getBase(), F, BB); SPIRVType *BaseSPVTy = AC->getBase()->getType(); if (BaseSPVTy->isTypePointer() && BaseSPVTy->getPointerElementType()->isTypeCooperativeMatrixKHR()) { return mapValue(BV, transSPIRVBuiltinFromInst(AC, BB)); } Type *BaseTy = BaseSPVTy->isTypeVector() ? transType( BaseSPVTy->getVectorComponentType()->getPointerElementType()) : transType(BaseSPVTy->getPointerElementType()); auto Index = transValue(AC->getIndices(), F, BB); if (!AC->hasPtrIndex()) Index.insert(Index.begin(), getInt32(M, 0)); auto IsInbound = AC->isInBounds(); Value *V = nullptr; if (BB) { auto GEP = GetElementPtrInst::Create(BaseTy, Base, Index, BV->getName(), BB); GEP->setIsInBounds(IsInbound); V = GEP; } else { V = ConstantExpr::getGetElementPtr(BaseTy, dyn_cast(Base), Index, IsInbound); } return mapValue(BV, V); } case OpPtrEqual: case OpPtrNotEqual: { auto *BC = static_cast(BV); auto Ops = transValue(BC->getOperands(), F, BB); IRBuilder<> Builder(BB); Value *Op1 = Builder.CreatePtrToInt(Ops[0], Type::getInt64Ty(*Context)); Value *Op2 = Builder.CreatePtrToInt(Ops[1], Type::getInt64Ty(*Context)); const CmpInst::Predicate P = OC == OpPtrEqual ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE; Value *V = Builder.CreateICmp(P, Op1, Op2); return mapValue(BV, V); } case OpPtrDiff: { auto *BC = static_cast(BV); auto Ops = transValue(BC->getOperands(), F, BB); IRBuilder<> Builder(BB); Type *ElemTy = transType(BC->getOperands()[0]->getType()->getPointerElementType()); Value *V = Builder.CreatePtrDiff(ElemTy, Ops[0], Ops[1]); return mapValue(BV, V); } case OpCompositeConstruct: { auto CC = static_cast(BV); auto Constituents = transValue(CC->getOperands(), F, BB); std::vector CV; bool HasRtValues = false; for (const auto &I : Constituents) { auto *C = dyn_cast(I); CV.push_back(C); if (!HasRtValues && C == nullptr) HasRtValues = true; } switch (static_cast(BV->getType()->getOpCode())) { case OpTypeVector: { if (!HasRtValues) return mapValue(BV, ConstantVector::get(CV)); auto *VT = cast(transType(CC->getType())); Value *NewVec = ConstantVector::getSplat( VT->getElementCount(), PoisonValue::get(VT->getElementType())); for (size_t I = 0; I < Constituents.size(); I++) { NewVec = InsertElementInst::Create(NewVec, Constituents[I], getInt32(M, I), "", BB); } return mapValue(BV, NewVec); } case OpTypeArray: { auto *AT = cast(transType(CC->getType())); if (!HasRtValues) return mapValue(BV, ConstantArray::get(AT, CV)); AllocaInst *Alloca = new AllocaInst(AT, SPIRAS_Private, "", BB); // get pointer to the element of the array // store the result of argument for (size_t I = 0; I < Constituents.size(); I++) { auto *GEP = GetElementPtrInst::Create( AT, Alloca, {getInt32(M, 0), getInt32(M, I)}, "gep", BB); GEP->setIsInBounds(true); new StoreInst(Constituents[I], GEP, false, BB); } auto *Load = new LoadInst(AT, Alloca, "load", false, BB); return mapValue(BV, Load); } case OpTypeStruct: { auto *ST = cast(transType(CC->getType())); if (!HasRtValues) return mapValue(BV, ConstantStruct::get(ST, CV)); AllocaInst *Alloca = new AllocaInst(ST, SPIRAS_Private, "", BB); // get pointer to the element of structure // store the result of argument for (size_t I = 0; I < Constituents.size(); I++) { auto *GEP = GetElementPtrInst::Create( ST, Alloca, {getInt32(M, 0), getInt32(M, I)}, "gep", BB); GEP->setIsInBounds(true); new StoreInst(Constituents[I], GEP, false, BB); } auto *Load = new LoadInst(ST, Alloca, "load", false, BB); return mapValue(BV, Load); } case internal::OpTypeJointMatrixINTEL: case OpTypeCooperativeMatrixKHR: return mapValue(BV, transSPIRVBuiltinFromInst(CC, BB)); default: llvm_unreachable("Unhandled type!"); } } case OpCompositeExtract: { SPIRVCompositeExtract *CE = static_cast(BV); IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } if (CE->getComposite()->getType()->isTypeVector()) { assert(CE->getIndices().size() == 1 && "Invalid index"); return mapValue( BV, Builder.CreateExtractElement( transValue(CE->getComposite(), F, BB), ConstantInt::get(*Context, APInt(32, CE->getIndices()[0])), BV->getName())); } return mapValue( BV, Builder.CreateExtractValue(transValue(CE->getComposite(), F, BB), CE->getIndices(), BV->getName())); } case OpVectorExtractDynamic: { auto *VED = static_cast(BV); SPIRVValue *Vec = VED->getVector(); if (Vec->getType()->getOpCode() == internal::OpTypeJointMatrixINTEL) { return mapValue(BV, transSPIRVBuiltinFromInst(VED, BB)); } return mapValue( BV, ExtractElementInst::Create(transValue(Vec, F, BB), transValue(VED->getIndex(), F, BB), BV->getName(), BB)); } case OpCompositeInsert: { auto CI = static_cast(BV); IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } if (CI->getComposite()->getType()->isTypeVector()) { assert(CI->getIndices().size() == 1 && "Invalid index"); return mapValue( BV, Builder.CreateInsertElement( transValue(CI->getComposite(), F, BB), transValue(CI->getObject(), F, BB), ConstantInt::get(*Context, APInt(32, CI->getIndices()[0])), BV->getName())); } return mapValue( BV, Builder.CreateInsertValue(transValue(CI->getComposite(), F, BB), transValue(CI->getObject(), F, BB), CI->getIndices(), BV->getName())); } case OpVectorInsertDynamic: { auto *VID = static_cast(BV); SPIRVValue *Vec = VID->getVector(); if (Vec->getType()->getOpCode() == internal::OpTypeJointMatrixINTEL) { return mapValue(BV, transSPIRVBuiltinFromInst(VID, BB)); } return mapValue( BV, InsertElementInst::Create( transValue(Vec, F, BB), transValue(VID->getComponent(), F, BB), transValue(VID->getIndex(), F, BB), BV->getName(), BB)); } case OpVectorShuffle: { auto VS = static_cast(BV); std::vector Components; IntegerType *Int32Ty = IntegerType::get(*Context, 32); for (auto I : VS->getComponents()) { if (I == static_cast(-1)) Components.push_back(UndefValue::get(Int32Ty)); else Components.push_back(ConstantInt::get(Int32Ty, I)); } IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } Value *Vec1 = transValue(VS->getVector1(), F, BB); Value *Vec2 = transValue(VS->getVector2(), F, BB); auto *Vec1Ty = cast(Vec1->getType()); auto *Vec2Ty = cast(Vec2->getType()); if (Vec1Ty->getNumElements() != Vec2Ty->getNumElements()) { // LLVM's shufflevector requires that the two vector operands have the // same type; SPIR-V's OpVectorShuffle allows the vector operands to // differ in the number of components. Adjust for that by extending // the smaller vector. if (Vec1Ty->getNumElements() < Vec2Ty->getNumElements()) { Vec1 = extendVector(Vec1, Vec2Ty, Builder); // Extending Vec1 requires offsetting any Vec2 indices in Components by // the number of new elements. unsigned Offset = Vec2Ty->getNumElements() - Vec1Ty->getNumElements(); unsigned Vec2Start = Vec1Ty->getNumElements(); for (auto &C : Components) { if (auto *CI = dyn_cast(C)) { uint64_t V = CI->getZExtValue(); if (V >= Vec2Start) { // This is a Vec2 index; add the offset to it. C = ConstantInt::get(Int32Ty, V + Offset); } } } } else { Vec2 = extendVector(Vec2, Vec1Ty, Builder); } } return mapValue( BV, Builder.CreateShuffleVector( Vec1, Vec2, ConstantVector::get(Components), BV->getName())); } case OpBitReverse: { auto *BR = static_cast(BV); auto Ty = transType(BV->getType()); Function *intr = Intrinsic::getDeclaration(M, llvm::Intrinsic::bitreverse, Ty); auto *Call = CallInst::Create(intr, transValue(BR->getOperand(0), F, BB), BR->getName(), BB); return mapValue(BV, Call); } case OpFunctionCall: { SPIRVFunctionCall *BC = static_cast(BV); auto Call = CallInst::Create(transFunction(BC->getFunction()), transValue(BC->getArgumentValues(), F, BB), BC->getName(), BB); setCallingConv(Call); setAttrByCalledFunc(Call); return mapValue(BV, Call); } case OpAsmCallINTEL: return mapValue( BV, transAsmCallINTEL(static_cast(BV), F, BB)); case OpFunctionPointerCallINTEL: { SPIRVFunctionPointerCallINTEL *BC = static_cast(BV); auto *V = transValue(BC->getCalledValue(), F, BB); auto *SpirvFnTy = BC->getCalledValue()->getType()->getPointerElementType(); auto *FnTy = cast(transType(SpirvFnTy)); auto *Call = CallInst::Create( FnTy, V, transValue(BC->getArgumentValues(), F, BB), BC->getName(), BB); transFunctionPointerCallArgumentAttributes( BV, Call, static_cast(SpirvFnTy)); // Assuming we are calling a regular device function Call->setCallingConv(CallingConv::SPIR_FUNC); // Don't set attributes, because at translation time we don't know which // function exactly we are calling. return mapValue(BV, Call); } case OpAssumeTrueKHR: { IRBuilder<> Builder(BB); SPIRVAssumeTrueKHR *BC = static_cast(BV); Value *Condition = transValue(BC->getCondition(), F, BB); return mapValue(BV, Builder.CreateAssumption(Condition)); } case OpExpectKHR: { IRBuilder<> Builder(BB); SPIRVExpectKHRInstBase *BC = static_cast(BV); Type *RetTy = transType(BC->getType()); Value *Val = transValue(BC->getOperand(0), F, BB); Value *ExpVal = transValue(BC->getOperand(1), F, BB); return mapValue( BV, Builder.CreateIntrinsic(Intrinsic::expect, RetTy, {Val, ExpVal})); } case OpExtInst: { auto *ExtInst = static_cast(BV); switch (ExtInst->getExtSetKind()) { case SPIRVEIS_OpenCL: { auto *V = mapValue(BV, transOCLBuiltinFromExtInst(ExtInst, BB)); applyFPFastMathModeDecorations(BV, static_cast(V)); return V; } case SPIRVEIS_Debug: case SPIRVEIS_OpenCL_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_200: return mapValue(BV, DbgTran->transDebugIntrinsic(ExtInst, BB)); default: llvm_unreachable("Unknown extended instruction set!"); } } case OpSNegate: { IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } SPIRVUnary *BC = static_cast(BV); if (BV->getType()->isTypeCooperativeMatrixKHR()) { return mapValue(BV, transSPIRVBuiltinFromInst(BC, BB)); } auto Neg = Builder.CreateNeg(transValue(BC->getOperand(0), F, BB), BV->getName()); if (auto *NegInst = dyn_cast(Neg)) { applyNoIntegerWrapDecorations(BV, NegInst); } return mapValue(BV, Neg); } case OpFMod: { // translate OpFMod(a, b) to: // r = frem(a, b) // c = copysign(r, b) // needs_fixing = islessgreater(r, c) // result = needs_fixing ? r + b : c IRBuilder<> Builder(BB); SPIRVFMod *FMod = static_cast(BV); auto Dividend = transValue(FMod->getOperand(0), F, BB); auto Divisor = transValue(FMod->getOperand(1), F, BB); auto FRem = Builder.CreateFRem(Dividend, Divisor, "frem.res"); auto CopySign = Builder.CreateBinaryIntrinsic( llvm::Intrinsic::copysign, FRem, Divisor, nullptr, "copysign.res"); auto FAdd = Builder.CreateFAdd(FRem, Divisor, "fadd.res"); auto Cmp = Builder.CreateFCmpONE(FRem, CopySign, "cmp.res"); auto Select = Builder.CreateSelect(Cmp, FAdd, CopySign); return mapValue(BV, Select); } case OpSMod: { // translate OpSMod(a, b) to: // r = srem(a, b) // needs_fixing = ((a < 0) != (b < 0) && r != 0) // result = needs_fixing ? r + b : r IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } SPIRVSMod *SMod = static_cast(BV); auto Dividend = transValue(SMod->getOperand(0), F, BB); auto Divisor = transValue(SMod->getOperand(1), F, BB); auto SRem = Builder.CreateSRem(Dividend, Divisor, "srem.res"); auto Xor = Builder.CreateXor(Dividend, Divisor, "xor.res"); auto Zero = ConstantInt::getNullValue(Dividend->getType()); auto CmpSign = Builder.CreateICmpSLT(Xor, Zero, "cmpsign.res"); auto CmpSRem = Builder.CreateICmpNE(SRem, Zero, "cmpsrem.res"); auto Add = Builder.CreateNSWAdd(SRem, Divisor, "add.res"); auto Cmp = Builder.CreateAnd(CmpSign, CmpSRem, "cmp.res"); auto Select = Builder.CreateSelect(Cmp, Add, SRem); return mapValue(BV, Select); } case OpFNegate: { SPIRVUnary *BC = static_cast(BV); if (BV->getType()->isTypeCooperativeMatrixKHR()) { return mapValue(BV, transSPIRVBuiltinFromInst(BC, BB)); } auto Neg = UnaryOperator::CreateFNeg(transValue(BC->getOperand(0), F, BB), BV->getName(), BB); applyFPFastMathModeDecorations(BV, Neg); return mapValue(BV, Neg); } case OpNot: case OpLogicalNot: { IRBuilder<> Builder(*Context); if (BB) { Builder.SetInsertPoint(BB); } SPIRVUnary *BC = static_cast(BV); return mapValue(BV, Builder.CreateNot(transValue(BC->getOperand(0), F, BB), BV->getName())); } case OpAll: case OpAny: return mapValue(BV, transAllAny(static_cast(BV), BB)); case OpIsFinite: case OpIsInf: case OpIsNan: case OpIsNormal: case OpSignBitSet: return mapValue(BV, transRelational(static_cast(BV), BB)); case OpIAddCarry: { auto *BC = static_cast(BV); return mapValue(BV, transBuiltinFromInst("__spirv_IAddCarry", BC, BB)); } case OpISubBorrow: { auto *BC = static_cast(BV); return mapValue(BV, transBuiltinFromInst("__spirv_ISubBorrow", BC, BB)); } case OpSMulExtended: { auto *BC = static_cast(BV); return mapValue(BV, transBuiltinFromInst("__spirv_SMulExtended", BC, BB)); } case OpUMulExtended: { auto *BC = static_cast(BV); return mapValue(BV, transBuiltinFromInst("__spirv_UMulExtended", BC, BB)); } case OpGetKernelWorkGroupSize: case OpGetKernelPreferredWorkGroupSizeMultiple: return mapValue( BV, transWGSizeQueryBI(static_cast(BV), BB)); case OpGetKernelNDrangeMaxSubGroupSize: case OpGetKernelNDrangeSubGroupCount: return mapValue( BV, transSGSizeQueryBI(static_cast(BV), BB)); case OpFPGARegINTEL: { IRBuilder<> Builder(BB); SPIRVFPGARegINTELInstBase *BC = static_cast(BV); PointerType *Int8PtrTyPrivate = Type::getInt8PtrTy(*Context, SPIRAS_Private); IntegerType *Int32Ty = Type::getInt32Ty(*Context); Value *UndefInt8Ptr = UndefValue::get(Int8PtrTyPrivate); Value *UndefInt32 = UndefValue::get(Int32Ty); Constant *GS = Builder.CreateGlobalStringPtr(kOCLBuiltinName::FPGARegIntel); Type *Ty = transType(BC->getType()); Value *Val = transValue(BC->getOperand(0), F, BB); Value *ValAsArg = Val; Type *RetTy = Ty; auto IID = Intrinsic::annotation; if (!isa(Ty)) { // All scalar types can be bitcasted to a same-sized integer if (!isa(Ty) && !isa(Ty)) { RetTy = IntegerType::get(*Context, Ty->getPrimitiveSizeInBits()); ValAsArg = Builder.CreateBitCast(Val, RetTy); } // If pointer type or struct type else { IID = Intrinsic::ptr_annotation; auto *PtrTy = dyn_cast(Ty); if (PtrTy && (PtrTy->isOpaque() || isa(PtrTy->getNonOpaquePointerElementType()))) RetTy = PtrTy; // Whether a struct or a pointer to some other type, // bitcast to i8* else { RetTy = Int8PtrTyPrivate; ValAsArg = Builder.CreateBitCast(Val, Int8PtrTyPrivate); } Value *Args[] = {ValAsArg, GS, UndefInt8Ptr, UndefInt32, UndefInt8Ptr}; auto *IntrinsicCall = Builder.CreateIntrinsic(IID, RetTy, Args); return mapValue(BV, IntrinsicCall); } } Value *Args[] = {ValAsArg, GS, UndefInt8Ptr, UndefInt32}; auto *IntrinsicCall = Builder.CreateIntrinsic(IID, RetTy, Args); return mapValue(BV, IntrinsicCall); } case OpFixedSqrtINTEL: case OpFixedRecipINTEL: case OpFixedRsqrtINTEL: case OpFixedSinINTEL: case OpFixedCosINTEL: case OpFixedSinCosINTEL: case OpFixedSinPiINTEL: case OpFixedCosPiINTEL: case OpFixedSinCosPiINTEL: case OpFixedLogINTEL: case OpFixedExpINTEL: return mapValue( BV, transFixedPointInst(static_cast(BV), BB)); case OpArbitraryFloatCastINTEL: case OpArbitraryFloatCastFromIntINTEL: case OpArbitraryFloatCastToIntINTEL: case OpArbitraryFloatRecipINTEL: case OpArbitraryFloatRSqrtINTEL: case OpArbitraryFloatCbrtINTEL: case OpArbitraryFloatSqrtINTEL: case OpArbitraryFloatLogINTEL: case OpArbitraryFloatLog2INTEL: case OpArbitraryFloatLog10INTEL: case OpArbitraryFloatLog1pINTEL: case OpArbitraryFloatExpINTEL: case OpArbitraryFloatExp2INTEL: case OpArbitraryFloatExp10INTEL: case OpArbitraryFloatExpm1INTEL: case OpArbitraryFloatSinINTEL: case OpArbitraryFloatCosINTEL: case OpArbitraryFloatSinCosINTEL: case OpArbitraryFloatSinPiINTEL: case OpArbitraryFloatCosPiINTEL: case OpArbitraryFloatSinCosPiINTEL: case OpArbitraryFloatASinINTEL: case OpArbitraryFloatASinPiINTEL: case OpArbitraryFloatACosINTEL: case OpArbitraryFloatACosPiINTEL: case OpArbitraryFloatATanINTEL: case OpArbitraryFloatATanPiINTEL: return mapValue(BV, transArbFloatInst(static_cast(BV), BB)); case OpArbitraryFloatAddINTEL: case OpArbitraryFloatSubINTEL: case OpArbitraryFloatMulINTEL: case OpArbitraryFloatDivINTEL: case OpArbitraryFloatGTINTEL: case OpArbitraryFloatGEINTEL: case OpArbitraryFloatLTINTEL: case OpArbitraryFloatLEINTEL: case OpArbitraryFloatEQINTEL: case OpArbitraryFloatHypotINTEL: case OpArbitraryFloatATan2INTEL: case OpArbitraryFloatPowINTEL: case OpArbitraryFloatPowRINTEL: case OpArbitraryFloatPowNINTEL: return mapValue( BV, transArbFloatInst(static_cast(BV), BB, true)); case OpArithmeticFenceEXT: { IRBuilder<> Builder(BB); auto *BC = static_cast(BV); Type *RetTy = transType(BC->getType()); Value *Val = transValue(BC->getOperand(0), F, BB); return mapValue( BV, Builder.CreateIntrinsic(Intrinsic::arithmetic_fence, RetTy, Val)); } case internal::OpMaskedGatherINTEL: { IRBuilder<> Builder(BB); auto *Inst = static_cast(BV); Type *RetTy = transType(Inst->getType()); Value *PtrVector = transValue(Inst->getOperand(0), F, BB); uint32_t Alignment = Inst->getOpWord(1); Value *Mask = transValue(Inst->getOperand(2), F, BB); Value *FillEmpty = transValue(Inst->getOperand(3), F, BB); return mapValue(BV, Builder.CreateMaskedGather(RetTy, PtrVector, Align(Alignment), Mask, FillEmpty)); } case internal::OpMaskedScatterINTEL: { IRBuilder<> Builder(BB); auto *Inst = static_cast(BV); Value *InputVector = transValue(Inst->getOperand(0), F, BB); Value *PtrVector = transValue(Inst->getOperand(1), F, BB); uint32_t Alignment = Inst->getOpWord(2); Value *Mask = transValue(Inst->getOperand(3), F, BB); return mapValue(BV, Builder.CreateMaskedScatter(InputVector, PtrVector, Align(Alignment), Mask)); } default: { auto OC = BV->getOpCode(); if (isCmpOpCode(OC)) return mapValue(BV, transCmpInst(BV, BB, F)); if (OCLSPIRVBuiltinMap::rfind(OC, nullptr)) return mapValue(BV, transSPIRVBuiltinFromInst( static_cast(BV), BB)); if (isBinaryShiftLogicalBitwiseOpCode(OC) || isLogicalOpCode(OC)) return mapValue(BV, transShiftLogicalBitwiseInst(BV, BB, F)); if (isCvtOpCode(OC) && OC != OpGenericCastToPtrExplicit) { auto BI = static_cast(BV); Value *Inst = nullptr; if (BI->hasFPRoundingMode() || BI->isSaturatedConversion()) Inst = transSPIRVBuiltinFromInst(BI, BB); else Inst = transConvertInst(BV, F, BB); return mapValue(BV, Inst); } return mapValue( BV, transSPIRVBuiltinFromInst(static_cast(BV), BB)); } } } // Get meaningful suffix for adding at the end of the function name to avoid // ascending numerical suffixes. It is useful in situations, where the same // function is called twice or more in one basic block. So, the function name is // formed in the following way: [FuncName].[ReturnTy].[InputTy] static std::string getFuncAPIntSuffix(const Type *RetTy, const Type *In1Ty, const Type *In2Ty = nullptr) { std::stringstream Suffix; Suffix << ".i" << RetTy->getIntegerBitWidth() << ".i" << In1Ty->getIntegerBitWidth(); if (In2Ty) Suffix << ".i" << In2Ty->getIntegerBitWidth(); return Suffix.str(); } Value *SPIRVToLLVM::transFixedPointInst(SPIRVInstruction *BI, BasicBlock *BB) { // LLVM fixed point functions return value: // iN (arbitrary precision integer of N bits length) // Arguments: // A(iN), S(i1), I(i32), rI(i32), Quantization(i32), Overflow(i32) // If return value wider than 64 bit: // iN addrspace(4)* sret(iN), A(iN), S(i1), I(i32), rI(i32), // Quantization(i32), Overflow(i32) // SPIR-V fixed point instruction contains: // ResTy Res In Literal S Literal I Literal rI Literal Q Literal O Type *RetTy = transType(BI->getType()); auto Inst = static_cast(BI); Type *InTy = transType(Inst->getOperand(0)->getType()); IntegerType *Int32Ty = IntegerType::get(*Context, 32); IntegerType *Int1Ty = IntegerType::get(*Context, 1); SmallVector ArgTys; std::vector Args; Args.reserve(8); if (RetTy->getIntegerBitWidth() > 64) { llvm::PointerType *RetPtrTy = llvm::PointerType::get(RetTy, SPIRAS_Generic); Value *Alloca = new AllocaInst(RetTy, SPIRAS_Private, "", BB); Value *RetValPtr = new AddrSpaceCastInst(Alloca, RetPtrTy, "", BB); ArgTys.emplace_back(RetPtrTy); Args.emplace_back(RetValPtr); } ArgTys.insert(ArgTys.end(), {InTy, Int1Ty, Int32Ty, Int32Ty, Int32Ty, Int32Ty}); auto Words = Inst->getOpWords(); Args.emplace_back(transValue(Inst->getOperand(0), BB->getParent(), BB)); Args.emplace_back(ConstantInt::get(Int1Ty, Words[1])); for (int I = 2; I <= 5; I++) Args.emplace_back(ConstantInt::get(Int32Ty, Words[I])); Type *FuncRetTy = (RetTy->getIntegerBitWidth() <= 64) ? RetTy : Type::getVoidTy(*Context); FunctionType *FT = FunctionType::get(FuncRetTy, ArgTys, false); Op OpCode = Inst->getOpCode(); std::string FuncName = SPIRVFixedPointIntelMap::rmap(OpCode) + getFuncAPIntSuffix(RetTy, InTy); FunctionCallee FCallee = M->getOrInsertFunction(FuncName, FT); auto *Func = cast(FCallee.getCallee()); Func->setCallingConv(CallingConv::SPIR_FUNC); if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); if (RetTy->getIntegerBitWidth() <= 64) return CallInst::Create(FCallee, Args, "", BB); Func->addParamAttr( 0, Attribute::get(*Context, Attribute::AttrKind::StructRet, RetTy)); CallInst *APIntInst = CallInst::Create(FCallee, Args, "", BB); APIntInst->addParamAttr( 0, Attribute::get(*Context, Attribute::AttrKind::StructRet, RetTy)); return static_cast(new LoadInst(RetTy, Args[0], "", false, BB)); } Value *SPIRVToLLVM::transArbFloatInst(SPIRVInstruction *BI, BasicBlock *BB, bool IsBinaryInst) { // Format of instructions Add, Sub, Mul, Div, Hypot, ATan2, Pow, PowR: // LLVM arbitrary floating point functions return value: // iN (arbitrary precision integer of N bits length) // Arguments: A(iN), MA(i32), B(iN), MB(i32), Mout(i32), // EnableSubnormals(i32), RoundingMode(i32), // RoundingAccuracy(i32) // where A, B and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal MB Literal Mout // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instructions GT, GE, LT, LE, EQ: // LLVM arbitrary floating point functions return value: Bool // Arguments: A(iN), MA(i32), B(iN), MB(i32) // where A and B are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal MB // Format of instruction PowN: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), B(iN), SignOfB(i1), Mout(i32), // EnableSubnormals(i32), RoundingMode(i32), // RoundingAccuracy(i32) // where A, B and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal SignOfB Literal Mout // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instruction CastFromInt: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), Mout(i32), FromSign(bool), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal Mout Literal FromSign // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instruction CastToInt: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), ToSign(bool), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA Literal ToSign // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of other instructions: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), Mout(i32), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA Literal Mout Literal EnableSubnormals // Literal RoundingMode Literal RoundingAccuracy Type *RetTy = transType(BI->getType()); IntegerType *Int1Ty = Type::getInt1Ty(*Context); IntegerType *Int32Ty = Type::getInt32Ty(*Context); auto Inst = static_cast(BI); Type *ATy = transType(Inst->getOperand(0)->getType()); Type *BTy = nullptr; // Words contain: // A [Literal MA] [B] [Literal MB] [Literal Mout] [Literal Sign] // [Literal EnableSubnormals Literal RoundingMode Literal RoundingAccuracy] const std::vector Words = Inst->getOpWords(); auto WordsItr = Words.begin() + 1; /* Skip word for A input id */ SmallVector ArgTys; std::vector Args; if (RetTy->getIntegerBitWidth() > 64) { llvm::PointerType *RetPtrTy = llvm::PointerType::get(RetTy, SPIRAS_Generic); ArgTys.push_back(RetPtrTy); Value *Alloca = new AllocaInst(RetTy, SPIRAS_Private, "", BB); Value *RetValPtr = new AddrSpaceCastInst(Alloca, RetPtrTy, "", BB); Args.push_back(RetValPtr); } ArgTys.insert(ArgTys.end(), {ATy, Int32Ty}); // A - input Args.emplace_back(transValue(Inst->getOperand(0), BB->getParent(), BB)); // MA/Mout - width of mantissa Args.emplace_back(ConstantInt::get(Int32Ty, *WordsItr++)); Op OC = Inst->getOpCode(); if (OC == OpArbitraryFloatCastFromIntINTEL || OC == OpArbitraryFloatCastToIntINTEL) { ArgTys.push_back(Int1Ty); Args.push_back(ConstantInt::get(Int1Ty, *WordsItr++)); /* ToSign/FromSign */ } if (IsBinaryInst) { /* B - input */ BTy = transType(Inst->getOperand(2)->getType()); ArgTys.push_back(BTy); Args.push_back(transValue(Inst->getOperand(2), BB->getParent(), BB)); ++WordsItr; /* Skip word for B input id */ if (OC == OpArbitraryFloatPowNINTEL) { ArgTys.push_back(Int1Ty); Args.push_back(ConstantInt::get(Int1Ty, *WordsItr++)); /* SignOfB */ } } std::fill_n(std::back_inserter(ArgTys), Words.end() - WordsItr, Int32Ty); std::transform(WordsItr, Words.end(), std::back_inserter(Args), [Int32Ty](const SPIRVWord &Word) { return ConstantInt::get(Int32Ty, Word); }); std::string FuncName = SPIRVArbFloatIntelMap::rmap(OC) + getFuncAPIntSuffix(RetTy, ATy, BTy); Type *FuncRetTy = (RetTy->getIntegerBitWidth() <= 64) ? RetTy : Type::getVoidTy(*Context); FunctionType *FT = FunctionType::get(FuncRetTy, ArgTys, false); FunctionCallee FCallee = M->getOrInsertFunction(FuncName, FT); auto *Func = cast(FCallee.getCallee()); Func->setCallingConv(CallingConv::SPIR_FUNC); if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); if (RetTy->getIntegerBitWidth() <= 64) return CallInst::Create(Func, Args, "", BB); Func->addParamAttr( 0, Attribute::get(*Context, Attribute::AttrKind::StructRet, RetTy)); CallInst *APFloatInst = CallInst::Create(FCallee, Args, "", BB); APFloatInst->addParamAttr( 0, Attribute::get(*Context, Attribute::AttrKind::StructRet, RetTy)); return static_cast(new LoadInst(RetTy, Args[0], "", false, BB)); } template bool SPIRVToLLVM::foreachFuncCtlMask(SourceTy Source, FuncTy Func) { SPIRVWord FCM = Source->getFuncCtlMask(); SPIRSPIRVFuncCtlMaskMap::foreach ( [&](Attribute::AttrKind Attr, SPIRVFunctionControlMaskKind Mask) { if (FCM & Mask) Func(Attr); }); return true; } void SPIRVToLLVM::transFunctionAttrs(SPIRVFunction *BF, Function *F) { if (BF->hasDecorate(DecorationReferencedIndirectlyINTEL)) F->addFnAttr("referenced-indirectly"); if (isFuncNoUnwind()) F->addFnAttr(Attribute::NoUnwind); foreachFuncCtlMask(BF, [&](Attribute::AttrKind Attr) { F->addFnAttr(Attr); }); for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { auto *BA = BF->getArgument(I->getArgNo()); mapValue(BA, &(*I)); setName(&(*I), BA); BA->foreachAttr([&](SPIRVFuncParamAttrKind Kind) { // Skip this function parameter attribute as it will translated among // OpenCL metadata if (Kind == FunctionParameterAttributeRuntimeAlignedINTEL) return; Attribute::AttrKind LLVMKind = SPIRSPIRVFuncParamAttrMap::rmap(Kind); Type *AttrTy = nullptr; switch (LLVMKind) { case Attribute::AttrKind::ByVal: case Attribute::AttrKind::StructRet: AttrTy = transType(BA->getType()->getPointerElementType()); break; default: break; // do nothing } // Make sure to use a correct constructor for a typed/typeless attribute auto A = AttrTy ? Attribute::get(*Context, LLVMKind, AttrTy) : Attribute::get(*Context, LLVMKind); I->addAttr(A); }); AttrBuilder Builder(*Context); SPIRVWord MaxOffset = 0; if (BA->hasDecorate(DecorationMaxByteOffset, 0, &MaxOffset)) Builder.addDereferenceableAttr(MaxOffset); else { SPIRVId MaxOffsetId; if (BA->hasDecorateId(DecorationMaxByteOffsetId, 0, &MaxOffsetId)) { if (auto MaxOffsetVal = transIdAsConstant(MaxOffsetId)) { Builder.addDereferenceableAttr(*MaxOffsetVal); } } } if (auto Alignment = getAlignment(BA)) { Builder.addAlignmentAttr(*Alignment); } I->addAttrs(Builder); } BF->foreachReturnValueAttr([&](SPIRVFuncParamAttrKind Kind) { if (Kind == FunctionParameterAttributeNoWrite) return; F->addRetAttr(SPIRSPIRVFuncParamAttrMap::rmap(Kind)); }); } namespace { // One basic block can be a predecessor to another basic block more than // once (https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/2702). // This function fixes any PHIs that break this rule. static void validatePhiPredecessors(Function *F) { for (BasicBlock &BB : *F) { bool UniquePreds = true; DenseMap PredsCnt; for (BasicBlock *PredBB : predecessors(&BB)) { auto It = PredsCnt.try_emplace(PredBB, 1); if (!It.second) { UniquePreds = false; ++It.first->second; } } if (UniquePreds) continue; // `phi` requires an incoming value per each predecessor instance, even // it's the same basic block that has been already inserted as an incoming // value of the `phi`. for (PHINode &Phi : BB.phis()) { SmallVector Vs; SmallVector Bs; SmallPtrSet UsedB; auto Vals = Phi.incoming_values(); auto Blocks = Phi.blocks(); auto *VIt = Vals.begin(); auto *BIt = Blocks.begin(); for (; VIt != Vals.end() && BIt != Blocks.end(); ++VIt, ++BIt) { const unsigned N = PredsCnt[*BIt]; if (!UsedB.insert(*BIt).second) continue; Vs.insert(Vs.end(), N, *VIt); Bs.insert(Bs.end(), N, *BIt); } unsigned I = 0; for (const unsigned N = Phi.getNumIncomingValues(); I < N; ++I) { Phi.setIncomingValue(I, Vs[I]); Phi.setIncomingBlock(I, Bs[I]); } for (const unsigned N = Vs.size(); I < N; ++I) Phi.addIncoming(Vs[I], Bs[I]); } } } } // namespace Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { auto Loc = FuncMap.find(BF); if (Loc != FuncMap.end()) return Loc->second; auto IsKernel = isKernel(BF); if (IsKernel) { // search for a previous function with the same name // upgrade it to a kernel and drop this if it's found for (auto &I : FuncMap) { auto BFName = I.getFirst()->getName(); if (BF->getName() == BFName) { auto *F = I.getSecond(); F->setCallingConv(CallingConv::SPIR_KERNEL); F->setLinkage(GlobalValue::ExternalLinkage); F->setDSOLocal(false); F = cast(mapValue(BF, F)); mapFunction(BF, F); transFunctionAttrs(BF, F); return F; } } } auto Linkage = IsKernel ? GlobalValue::ExternalLinkage : transLinkageType(BF); FunctionType *FT = dyn_cast(transType(BF->getFunctionType())); std::string FuncName = BF->getName(); StringRef FuncNameRef(FuncName); // Transform "@spirv.llvm_memset_p0i8_i32.volatile" to @llvm.memset.p0i8.i32 // assuming llvm.memset is supported by the device compiler. If this // assumption is not safe, we should have a command line option to control // this behavior. if (FuncNameRef.startswith("spirv.llvm_memset_p")) { // We can't guarantee that the name is correctly mangled due to opaque // pointers. Derive the correct name from the function type. FuncName = Intrinsic::getDeclaration(M, Intrinsic::memset, {FT->getParamType(0), FT->getParamType(2)}) ->getName() .str(); } if (FuncNameRef.consume_front("spirv.")) { FuncNameRef.consume_back(".volatile"); FuncName = FuncNameRef.str(); std::replace(FuncName.begin(), FuncName.end(), '_', '.'); } Function *F = M->getFunction(FuncName); if (!F) F = Function::Create(FT, Linkage, AS, FuncName, M); F = cast(mapValue(BF, F)); mapFunction(BF, F); if (F->isIntrinsic()) { if (F->getIntrinsicID() != Intrinsic::umul_with_overflow) return F; std::string Name = F->getName().str(); auto *ST = dyn_cast(F->getReturnType()); auto *FT = F->getFunctionType(); auto *NewST = StructType::get(ST->getContext(), ST->elements()); auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg()); F->setName("old_" + Name); auto *NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(), Name, F->getParent()); return NewFn; } F->setCallingConv(IsKernel ? CallingConv::SPIR_KERNEL : CallingConv::SPIR_FUNC); transFunctionAttrs(BF, F); // Creating all basic blocks before creating instructions. for (size_t I = 0, E = BF->getNumBasicBlock(); I != E; ++I) { transValue(BF->getBasicBlock(I), F, nullptr); } for (size_t I = 0, E = BF->getNumBasicBlock(); I != E; ++I) { SPIRVBasicBlock *BBB = BF->getBasicBlock(I); BasicBlock *BB = dyn_cast(transValue(BBB, F, nullptr)); for (size_t BI = 0, BE = BBB->getNumInst(); BI != BE; ++BI) { SPIRVInstruction *BInst = BBB->getInst(BI); transValue(BInst, F, BB, false); } } validatePhiPredecessors(F); transLLVMLoopMetadata(F); return F; } Value *SPIRVToLLVM::transAsmINTEL(SPIRVAsmINTEL *BA) { assert(BA); bool HasSideEffect = BA->hasDecorate(DecorationSideEffectsINTEL); return InlineAsm::get( cast(transType(BA->getFunctionType())), BA->getInstructions(), BA->getConstraints(), HasSideEffect, /* IsAlignStack */ false, InlineAsm::AsmDialect::AD_ATT); } CallInst *SPIRVToLLVM::transAsmCallINTEL(SPIRVAsmCallINTEL *BI, Function *F, BasicBlock *BB) { assert(BI); auto *IA = cast(transValue(BI->getAsm(), F, BB)); auto Args = transValue(BM->getValues(BI->getArguments()), F, BB); return CallInst::Create(cast(IA->getFunctionType()), IA, Args, BI->getName(), BB); } /// LLVM convert builtin functions is translated to two instructions: /// y = i32 islessgreater(float x, float z) -> /// y = i32 ZExt(bool LessOrGreater(float x, float z)) /// When translating back, for simplicity, a trunc instruction is inserted /// w = bool LessOrGreater(float x, float z) -> /// w = bool Trunc(i32 islessgreater(float x, float z)) /// Optimizer should be able to remove the redundant trunc/zext void SPIRVToLLVM::transOCLBuiltinFromInstPreproc( SPIRVInstruction *BI, Type *&RetTy, std::vector &Args) { if (!BI->hasType()) return; auto BT = BI->getType(); if (isCmpOpCode(BI->getOpCode())) { if (BT->isTypeBool()) RetTy = IntegerType::getInt32Ty(*Context); else if (BT->isTypeVectorBool()) RetTy = FixedVectorType::get( IntegerType::get( *Context, Args[0]->getType()->getVectorComponentType()->getBitWidth()), BT->getVectorComponentCount()); else llvm_unreachable("invalid compare instruction"); } } Instruction * SPIRVToLLVM::transOCLBuiltinPostproc(SPIRVInstruction *BI, CallInst *CI, BasicBlock *BB, const std::string &DemangledName) { auto OC = BI->getOpCode(); if (isCmpOpCode(OC) && BI->getType()->isTypeVectorOrScalarBool()) { return CastInst::Create(Instruction::Trunc, CI, transType(BI->getType()), "cvt", BB); } if (SPIRVEnableStepExpansion && (DemangledName == "smoothstep" || DemangledName == "step")) return expandOCLBuiltinWithScalarArg(CI, DemangledName); return CI; } Value *SPIRVToLLVM::transBlockInvoke(SPIRVValue *Invoke, BasicBlock *BB) { auto *TranslatedInvoke = transFunction(static_cast(Invoke)); auto *Int8PtrTyGen = Type::getInt8PtrTy(*Context, SPIRAS_Generic); return CastInst::CreatePointerBitCastOrAddrSpaceCast(TranslatedInvoke, Int8PtrTyGen, "", BB); } Instruction *SPIRVToLLVM::transWGSizeQueryBI(SPIRVInstruction *BI, BasicBlock *BB) { std::string FName = (BI->getOpCode() == OpGetKernelWorkGroupSize) ? "__get_kernel_work_group_size_impl" : "__get_kernel_preferred_work_group_size_multiple_impl"; Function *F = M->getFunction(FName); if (!F) { auto Int8PtrTyGen = Type::getInt8PtrTy(*Context, SPIRAS_Generic); FunctionType *FT = FunctionType::get(Type::getInt32Ty(*Context), {Int8PtrTyGen, Int8PtrTyGen}, false); F = Function::Create(FT, GlobalValue::ExternalLinkage, FName, M); if (isFuncNoUnwind()) F->addFnAttr(Attribute::NoUnwind); } auto Ops = BI->getOperands(); SmallVector Args = {transBlockInvoke(Ops[0], BB), transValue(Ops[1], F, BB, false)}; auto Call = CallInst::Create(F, Args, "", BB); setName(Call, BI); setAttrByCalledFunc(Call); return Call; } Instruction *SPIRVToLLVM::transSGSizeQueryBI(SPIRVInstruction *BI, BasicBlock *BB) { std::string FName = (BI->getOpCode() == OpGetKernelNDrangeMaxSubGroupSize) ? "__get_kernel_max_sub_group_size_for_ndrange_impl" : "__get_kernel_sub_group_count_for_ndrange_impl"; auto Ops = BI->getOperands(); Function *F = M->getFunction(FName); if (!F) { auto Int8PtrTyGen = Type::getInt8PtrTy(*Context, SPIRAS_Generic); SmallVector Tys = { transType(Ops[0]->getType()), // ndrange Int8PtrTyGen, // block_invoke Int8PtrTyGen // block_literal }; auto *FT = FunctionType::get(Type::getInt32Ty(*Context), Tys, false); F = Function::Create(FT, GlobalValue::ExternalLinkage, FName, M); if (isFuncNoUnwind()) F->addFnAttr(Attribute::NoUnwind); } SmallVector Args = { transValue(Ops[0], F, BB, false), // ndrange transBlockInvoke(Ops[1], BB), // block_invoke transValue(Ops[2], F, BB, false) // block_literal }; auto Call = CallInst::Create(F, Args, "", BB); setName(Call, BI); setAttrByCalledFunc(Call); return Call; } Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, SPIRVInstruction *BI, BasicBlock *BB) { std::string MangledName; auto Ops = BI->getOperands(); Type *RetTy = BI->hasType() ? transType(BI->getType()) : Type::getVoidTy(*Context); transOCLBuiltinFromInstPreproc(BI, RetTy, Ops); std::vector ArgTys = transTypeVector(SPIRVInstruction::getOperandTypes(Ops)); for (auto &I : ArgTys) { if (isa(I)) { I = PointerType::get(I, SPIRAS_Private); } } std::vector PointerElementTys = getPointerElementTypes(SPIRVInstruction::getOperandTypes(Ops)); if (BM->getDesiredBIsRepresentation() != BIsRepresentation::SPIRVFriendlyIR) mangleOpenClBuiltin(FuncName, ArgTys, PointerElementTys, MangledName); else MangledName = getSPIRVFriendlyIRFunctionName( FuncName, BI->getOpCode(), ArgTys, PointerElementTys, Ops); Function *Func = M->getFunction(MangledName); FunctionType *FT = FunctionType::get(RetTy, ArgTys, false); // ToDo: Some intermediate functions have duplicate names with // different function types. This is OK if the function name // is used internally and finally translated to unique function // names. However it is better to have a way to differentiate // between intermidiate functions and final functions and make // sure final functions have unique names. SPIRVDBG(if (Func && Func->getFunctionType() != FT) { dbgs() << "Warning: Function name conflict:\n" << *Func << '\n' << " => " << *FT << '\n'; }) if (!Func || Func->getFunctionType() != FT) { LLVM_DEBUG(for (auto &I : ArgTys) { dbgs() << *I << '\n'; }); Func = Function::Create(FT, GlobalValue::ExternalLinkage, MangledName, M); Func->setCallingConv(CallingConv::SPIR_FUNC); if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); auto OC = BI->getOpCode(); if (isGroupOpCode(OC) || isIntelSubgroupOpCode(OC) || isSplitBarrierINTELOpCode(OC) || OC == OpControlBarrier) Func->addFnAttr(Attribute::Convergent); } CallInst *Call; if (BI->getOpCode() == OpCooperativeMatrixLengthKHR && Ops[0]->getOpCode() == OpTypeCooperativeMatrixKHR) { // OpCooperativeMatrixLengthKHR needs special handling as its operand is // a Type instead of a Value. llvm::Type *MatTy = transType(reinterpret_cast(Ops[0])); Call = CallInst::Create(Func, Constant::getNullValue(MatTy), "", BB); } else { Call = CallInst::Create(Func, transValue(Ops, BB->getParent(), BB), "", BB); } setName(Call, BI); setAttrByCalledFunc(Call); SPIRVDBG(spvdbgs() << "[transInstToBuiltinCall] " << *BI << " -> "; dbgs() << *Call << '\n';) Instruction *Inst = transOCLBuiltinPostproc(BI, Call, BB, FuncName); return Inst; } SPIRVToLLVM::SPIRVToLLVM(Module *LLVMModule, SPIRVModule *TheSPIRVModule) : M(LLVMModule), BM(TheSPIRVModule) { assert(M && "Initialization without an LLVM module is not allowed"); Context = &M->getContext(); DbgTran.reset(new SPIRVToLLVMDbgTran(TheSPIRVModule, LLVMModule, this)); } std::string getSPIRVFuncSuffix(SPIRVInstruction *BI) { string Suffix = ""; if (BI->getOpCode() == OpCreatePipeFromPipeStorage) { auto CPFPS = static_cast(BI); assert(CPFPS->getType()->isTypePipe() && "Invalid type of CreatePipeFromStorage"); auto PipeType = static_cast(CPFPS->getType()); switch (PipeType->getAccessQualifier()) { default: case AccessQualifierReadOnly: Suffix = "_read"; break; case AccessQualifierWriteOnly: Suffix = "_write"; break; case AccessQualifierReadWrite: Suffix = "_read_write"; break; } } if (BI->hasDecorate(DecorationSaturatedConversion)) { Suffix += kSPIRVPostfix::Divider; Suffix += kSPIRVPostfix::Sat; } SPIRVFPRoundingModeKind Kind; if (BI->hasFPRoundingMode(&Kind)) { Suffix += kSPIRVPostfix::Divider; Suffix += SPIRSPIRVFPRoundingModeMap::rmap(Kind); } if (BI->getOpCode() == OpGenericCastToPtrExplicit) { Suffix += kSPIRVPostfix::Divider; auto *Ty = BI->getType(); auto GenericCastToPtrInst = Ty->isTypeVectorPointer() ? Ty->getVectorComponentType()->getPointerStorageClass() : Ty->getPointerStorageClass(); switch (GenericCastToPtrInst) { case StorageClassCrossWorkgroup: Suffix += std::string(kSPIRVPostfix::ToGlobal); break; case StorageClassWorkgroup: Suffix += std::string(kSPIRVPostfix::ToLocal); break; case StorageClassFunction: Suffix += std::string(kSPIRVPostfix::ToPrivate); break; default: llvm_unreachable("Invalid address space"); } } if (BI->getOpCode() == OpBuildNDRange) { Suffix += kSPIRVPostfix::Divider; auto *NDRangeInst = static_cast(BI); auto *EleTy = ((NDRangeInst->getOperands())[0])->getType(); int Dim = EleTy->isTypeArray() ? EleTy->getArrayLength() : 1; assert((EleTy->isTypeInt() && Dim == 1) || (EleTy->isTypeArray() && Dim >= 2 && Dim <= 3)); ostringstream OS; OS << Dim; Suffix += OS.str() + "D"; } return Suffix; } Instruction *SPIRVToLLVM::transSPIRVBuiltinFromInst(SPIRVInstruction *BI, BasicBlock *BB) { assert(BB && "Invalid BB"); const auto OC = BI->getOpCode(); bool AddRetTypePostfix = false; switch (static_cast(OC)) { case OpImageQuerySizeLod: case OpImageQuerySize: case OpImageRead: case OpSubgroupImageBlockReadINTEL: case OpSubgroupImageMediaBlockReadINTEL: case OpSubgroupBlockReadINTEL: case OpImageSampleExplicitLod: case OpSDotKHR: case OpUDotKHR: case OpSUDotKHR: case OpSDotAccSatKHR: case OpUDotAccSatKHR: case OpSUDotAccSatKHR: case OpReadClockKHR: case internal::OpJointMatrixLoadINTEL: case OpCooperativeMatrixLoadKHR: case internal::OpCooperativeMatrixLoadCheckedINTEL: case internal::OpConvertHandleToImageINTEL: case internal::OpConvertHandleToSampledImageINTEL: AddRetTypePostfix = true; break; default: { if (isCvtOpCode(OC) && OC != OpGenericCastToPtrExplicit) AddRetTypePostfix = true; break; } } bool IsRetSigned = true; switch (OC) { case OpConvertFToU: case OpSatConvertSToU: case OpUConvert: case OpUDotKHR: case OpUDotAccSatKHR: case OpReadClockKHR: IsRetSigned = false; break; case OpImageRead: case OpImageSampleExplicitLod: { const size_t Idx = getImageOperandsIndex(OC); auto Ops = BI->getOperands(); if (Ops.size() > Idx) { auto ImOp = static_cast(Ops[Idx])->getZExtIntValue(); IsRetSigned = !(ImOp & ImageOperandsMask::ImageOperandsZeroExtendMask); } break; } default: break; } if (AddRetTypePostfix) { const Type *RetTy = BI->hasType() ? transType(BI->getType()) : Type::getVoidTy(*Context); Type *PET = RetTy->isPointerTy() ? getPointerElementTypes(BI->getType())[0].getPointer() : nullptr; return transBuiltinFromInst(getSPIRVFuncName(OC, RetTy, IsRetSigned, PET) + getSPIRVFuncSuffix(BI), BI, BB); } return transBuiltinFromInst(getSPIRVFuncName(OC, getSPIRVFuncSuffix(BI)), BI, BB); } bool SPIRVToLLVM::translate() { if (!transAddressingModel()) return false; // Entry Points should be translated before all debug intrinsics. for (SPIRVExtInst *EI : BM->getDebugInstVec()) { if (EI->getExtOp() == SPIRVDebug::EntryPoint) DbgTran->transDebugInst(EI); } // Compile unit might be needed during translation of debug intrinsics. for (SPIRVExtInst *EI : BM->getDebugInstVec()) { // Translate Compile Units first. if (EI->getExtOp() == SPIRVDebug::CompilationUnit) DbgTran->transDebugInst(EI); } for (unsigned I = 0, E = BM->getNumVariables(); I != E; ++I) { auto *BV = BM->getVariable(I); if (BV->getStorageClass() != StorageClassFunction) transValue(BV, nullptr, nullptr); else transGlobalCtorDtors(BV); } // Then translate all debug instructions. for (SPIRVExtInst *EI : BM->getDebugInstVec()) { DbgTran->transDebugInst(EI); } for (auto *FP : BM->getFunctionPointers()) { SPIRVConstantFunctionPointerINTEL *BC = static_cast(FP); SPIRVFunction *F = BC->getFunction(); FP->setName(F->getName()); const unsigned AS = BM->shouldEmitFunctionPtrAddrSpace() ? SPIRAS_CodeSectionINTEL : SPIRAS_Private; mapValue(FP, transFunction(F, AS)); } for (unsigned I = 0, E = BM->getNumFunctions(); I != E; ++I) { transFunction(BM->getFunction(I)); transUserSemantic(BM->getFunction(I)); } transGlobalAnnotations(); if (!transMetadata()) return false; if (!transFPContractMetadata()) return false; transSourceLanguage(); if (!transSourceExtension()) return false; transGeneratorMD(); if (!lowerBuiltins(BM, M)) return false; if (BM->getDesiredBIsRepresentation() == BIsRepresentation::SPIRVFriendlyIR) { SPIRVWord SrcLangVer = 0; BM->getSourceLanguage(&SrcLangVer); bool IsCpp = SrcLangVer == kOCLVer::CL21; if (!postProcessBuiltinsReturningStruct(M, IsCpp)) return false; } for (SPIRVExtInst *EI : BM->getAuxDataInstVec()) { transAuxDataInst(EI); } eraseUselessFunctions(M); DbgTran->addDbgInfoVersion(); DbgTran->finalize(); return true; } bool SPIRVToLLVM::transAddressingModel() { switch (BM->getAddressingModel()) { case AddressingModelPhysical64: M->setTargetTriple(SPIR_TARGETTRIPLE64); M->setDataLayout(SPIR_DATALAYOUT64); break; case AddressingModelPhysical32: M->setTargetTriple(SPIR_TARGETTRIPLE32); M->setDataLayout(SPIR_DATALAYOUT32); break; case AddressingModelLogical: // Do not set target triple and data layout break; default: SPIRVCKRT(0, InvalidAddressingModel, "Actual addressing mode is " + std::to_string(BM->getAddressingModel())); } return true; } void generateIntelFPGAAnnotation(const SPIRVEntry *E, llvm::SmallString<256> &AnnotStr) { llvm::raw_svector_ostream Out(AnnotStr); if (E->hasDecorate(DecorationRegisterINTEL)) Out << "{register:1}"; SPIRVWord Result = 0; if (E->hasDecorate(DecorationMemoryINTEL)) Out << "{memory:" << E->getDecorationStringLiteral(DecorationMemoryINTEL).front() << '}'; if (E->hasDecorate(DecorationBankwidthINTEL, 0, &Result)) Out << "{bankwidth:" << Result << '}'; if (E->hasDecorate(DecorationNumbanksINTEL, 0, &Result)) Out << "{numbanks:" << Result << '}'; if (E->hasDecorate(DecorationMaxPrivateCopiesINTEL, 0, &Result)) Out << "{private_copies:" << Result << '}'; if (E->hasDecorate(DecorationSinglepumpINTEL)) Out << "{pump:1}"; if (E->hasDecorate(DecorationDoublepumpINTEL)) Out << "{pump:2}"; if (E->hasDecorate(DecorationMaxReplicatesINTEL, 0, &Result)) Out << "{max_replicates:" << Result << '}'; if (E->hasDecorate(DecorationSimpleDualPortINTEL)) Out << "{simple_dual_port:1}"; if (E->hasDecorate(DecorationMergeINTEL)) { Out << "{merge"; for (auto Str : E->getDecorationStringLiteral(DecorationMergeINTEL)) Out << ":" << Str; Out << '}'; } if (E->hasDecorate(DecorationBankBitsINTEL)) { Out << "{bank_bits:"; auto Literals = E->getDecorationLiterals(DecorationBankBitsINTEL); for (size_t I = 0; I < Literals.size() - 1; ++I) Out << Literals[I] << ","; Out << Literals.back() << '}'; } if (E->hasDecorate(DecorationForcePow2DepthINTEL, 0, &Result)) Out << "{force_pow2_depth:" << Result << '}'; if (E->hasDecorate(DecorationUserSemantic)) Out << E->getDecorationStringLiteral(DecorationUserSemantic).front(); unsigned LSUParamsBitmask = 0; llvm::SmallString<32> AdditionalParamsStr; llvm::raw_svector_ostream ParamsOut(AdditionalParamsStr); if (E->hasDecorate(DecorationBurstCoalesceINTEL, 0)) LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::BurstCoalesce; if (E->hasDecorate(DecorationCacheSizeINTEL, 0, &Result)) { LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::CacheSizeFlag; ParamsOut << "{cache-size:" << Result << "}"; } if (E->hasDecorate(DecorationDontStaticallyCoalesceINTEL, 0)) LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::DontStaticallyCoalesce; if (E->hasDecorate(DecorationPrefetchINTEL, 0, &Result)) { LSUParamsBitmask |= IntelFPGAMemoryAccessesVal::PrefetchFlag; // TODO: Enable prefetch size backwards translation // once it is supported } if (LSUParamsBitmask == 0) return; Out << "{params:" << LSUParamsBitmask << "}" << AdditionalParamsStr; } void generateIntelFPGAAnnotationForStructMember( const SPIRVEntry *E, SPIRVWord MemberNumber, llvm::SmallString<256> &AnnotStr) { llvm::raw_svector_ostream Out(AnnotStr); if (E->hasMemberDecorate(DecorationRegisterINTEL, 0, MemberNumber)) Out << "{register:1}"; SPIRVWord Result = 0; if (E->hasMemberDecorate(DecorationMemoryINTEL, 0, MemberNumber, &Result)) Out << "{memory:" << E->getMemberDecorationStringLiteral(DecorationMemoryINTEL, MemberNumber) .front() << '}'; if (E->hasMemberDecorate(DecorationBankwidthINTEL, 0, MemberNumber, &Result)) Out << "{bankwidth:" << Result << '}'; if (E->hasMemberDecorate(DecorationNumbanksINTEL, 0, MemberNumber, &Result)) Out << "{numbanks:" << Result << '}'; if (E->hasMemberDecorate(DecorationMaxPrivateCopiesINTEL, 0, MemberNumber, &Result)) Out << "{private_copies:" << Result << '}'; if (E->hasMemberDecorate(DecorationSinglepumpINTEL, 0, MemberNumber)) Out << "{pump:1}"; if (E->hasMemberDecorate(DecorationDoublepumpINTEL, 0, MemberNumber)) Out << "{pump:2}"; if (E->hasMemberDecorate(DecorationMaxReplicatesINTEL, 0, MemberNumber, &Result)) Out << "{max_replicates:" << Result << '}'; if (E->hasMemberDecorate(DecorationSimpleDualPortINTEL, 0, MemberNumber)) Out << "{simple_dual_port:1}"; if (E->hasMemberDecorate(DecorationMergeINTEL, 0, MemberNumber)) { Out << "{merge"; for (auto Str : E->getMemberDecorationStringLiteral(DecorationMergeINTEL, MemberNumber)) Out << ":" << Str; Out << '}'; } if (E->hasMemberDecorate(DecorationBankBitsINTEL, 0, MemberNumber)) { Out << "{bank_bits:"; auto Literals = E->getMemberDecorationLiterals(DecorationBankBitsINTEL, MemberNumber); for (size_t I = 0; I < Literals.size() - 1; ++I) Out << Literals[I] << ","; Out << Literals.back() << '}'; } if (E->hasMemberDecorate(DecorationForcePow2DepthINTEL, 0, MemberNumber, &Result)) Out << "{force_pow2_depth:" << Result << '}'; if (E->hasMemberDecorate(DecorationUserSemantic, 0, MemberNumber)) Out << E->getMemberDecorationStringLiteral(DecorationUserSemantic, MemberNumber) .front(); } void SPIRVToLLVM::transIntelFPGADecorations(SPIRVValue *BV, Value *V) { if (!BV->isVariable() && !BV->isInst()) return; if (auto *Inst = dyn_cast(V)) { auto *AL = dyn_cast(Inst); Type *AllocatedTy = AL ? AL->getAllocatedType() : Inst->getType(); IRBuilder<> Builder(Inst->getParent()); Type *Int8PtrTyPrivate = Type::getInt8PtrTy(*Context, SPIRAS_Private); IntegerType *Int32Ty = IntegerType::get(*Context, 32); Value *UndefInt8Ptr = UndefValue::get(Int8PtrTyPrivate); Value *UndefInt32 = UndefValue::get(Int32Ty); if (AL && BV->getType()->getPointerElementType()->isTypeStruct()) { auto *ST = BV->getType()->getPointerElementType(); SPIRVTypeStruct *STS = static_cast(ST); for (SPIRVWord I = 0; I < STS->getMemberCount(); ++I) { SmallString<256> AnnotStr; generateIntelFPGAAnnotationForStructMember(ST, I, AnnotStr); if (!AnnotStr.empty()) { auto *GS = Builder.CreateGlobalStringPtr(AnnotStr); auto *GEP = cast( Builder.CreateConstInBoundsGEP2_32(AllocatedTy, AL, 0, I)); Type *IntTy = GEP->getResultElementType()->isIntegerTy() ? GEP->getType() : Int8PtrTyPrivate; auto AnnotationFn = llvm::Intrinsic::getDeclaration( M, Intrinsic::ptr_annotation, IntTy); llvm::Value *Args[] = { Builder.CreateBitCast(GEP, IntTy, GEP->getName()), Builder.CreateBitCast(GS, Int8PtrTyPrivate), UndefInt8Ptr, UndefInt32, UndefInt8Ptr}; Builder.CreateCall(AnnotationFn, Args); } } } SmallString<256> AnnotStr; generateIntelFPGAAnnotation(BV, AnnotStr); if (!AnnotStr.empty()) { Constant *GS = nullptr; std::string StringAnnotStr = AnnotStr.c_str(); auto AnnotItr = AnnotationsMap.find(StringAnnotStr); if (AnnotItr != AnnotationsMap.end()) { GS = AnnotItr->second; } else { GS = Builder.CreateGlobalStringPtr(AnnotStr); AnnotationsMap.emplace(std::move(StringAnnotStr), GS); } Value *BaseInst = AL ? Builder.CreateBitCast(V, Int8PtrTyPrivate, V->getName()) : Inst; // Try to find alloca instruction for statically allocated variables. // Alloca might be hidden by a couple of casts. bool isStaticMemoryAttribute = AL ? true : false; while (!isStaticMemoryAttribute && Inst && (isa(Inst) || isa(Inst))) { Inst = dyn_cast(Inst->getOperand(0)); isStaticMemoryAttribute = (Inst && isa(Inst)); } auto AnnotationFn = isStaticMemoryAttribute ? llvm::Intrinsic::getDeclaration(M, Intrinsic::var_annotation) : llvm::Intrinsic::getDeclaration(M, Intrinsic::ptr_annotation, BaseInst->getType()); llvm::Value *Args[] = {BaseInst, Builder.CreateBitCast(GS, Int8PtrTyPrivate), UndefInt8Ptr, UndefInt32, UndefInt8Ptr}; Builder.CreateCall(AnnotationFn, Args); } } else if (auto *GV = dyn_cast(V)) { // Do not add annotations for builtin variables. SPIRVBuiltinVariableKind Kind; if (isSPIRVBuiltinVariable(GV, &Kind)) return; SmallString<256> AnnotStr; generateIntelFPGAAnnotation(BV, AnnotStr); if (AnnotStr.empty()) { // Check if IO pipe decoration is applied to the global SPIRVWord ID; if (BV->hasDecorate(DecorationIOPipeStorageINTEL, 0, &ID)) { auto Literals = BV->getDecorationLiterals(DecorationIOPipeStorageINTEL); assert(Literals.size() == 1 && "IO PipeStorage decoration shall have 1 extra operand"); GV->setMetadata("io_pipe_id", getMDNodeStringIntVec(Context, Literals)); } return; } Constant *StrConstant = ConstantDataArray::getString(*Context, StringRef(AnnotStr)); auto *GS = new GlobalVariable(*GV->getParent(), StrConstant->getType(), /*IsConstant*/ true, GlobalValue::PrivateLinkage, StrConstant, ""); GS->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); GS->setSection("llvm.metadata"); Type *ResType = PointerType::getInt8PtrTy( GV->getContext(), GV->getType()->getPointerAddressSpace()); Constant *C = ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, ResType); Type *Int8PtrTyPrivate = Type::getInt8PtrTy(*Context, SPIRAS_Private); IntegerType *Int32Ty = Type::getInt32Ty(*Context); llvm::Constant *Fields[5] = { C, ConstantExpr::getBitCast(GS, Int8PtrTyPrivate), UndefValue::get(Int8PtrTyPrivate), UndefValue::get(Int32Ty), UndefValue::get(Int8PtrTyPrivate)}; GlobalAnnotations.push_back(ConstantStruct::getAnon(Fields)); } } // Translate aliasing decorations applied to instructions. These decorations // are mapped on alias.scope and noalias metadata in LLVM. Translation of // optional string operand isn't yet supported in the translator. void SPIRVToLLVM::transMemAliasingINTELDecorations(SPIRVValue *BV, Value *V) { if (!BV->isInst()) return; Instruction *Inst = dyn_cast(V); if (!Inst) return; if (BV->hasDecorateId(DecorationAliasScopeINTEL)) { std::vector AliasListIds; AliasListIds = BV->getDecorationIdLiterals(DecorationAliasScopeINTEL); assert(AliasListIds.size() == 1 && "Memory aliasing decorations must have one argument"); addMemAliasMetadata(Inst, AliasListIds[0], LLVMContext::MD_alias_scope); } if (BV->hasDecorateId(DecorationNoAliasINTEL)) { std::vector AliasListIds; AliasListIds = BV->getDecorationIdLiterals(DecorationNoAliasINTEL); assert(AliasListIds.size() == 1 && "Memory aliasing decorations must have one argument"); addMemAliasMetadata(Inst, AliasListIds[0], LLVMContext::MD_noalias); } } // Having UserSemantic decoration on Function is against the spec, but we allow // this for various purposes (like prototyping new features when we need to // attach some information on function and propagate that through SPIR-V and // ect.) void SPIRVToLLVM::transUserSemantic(SPIRV::SPIRVFunction *Fun) { auto TransFun = transFunction(Fun); for (auto UsSem : Fun->getDecorationStringLiteral(DecorationUserSemantic)) { auto V = cast(TransFun); Constant *StrConstant = ConstantDataArray::getString(*Context, StringRef(UsSem)); auto *GS = new GlobalVariable( *TransFun->getParent(), StrConstant->getType(), /*IsConstant*/ true, GlobalValue::PrivateLinkage, StrConstant, ""); GS->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); GS->setSection("llvm.metadata"); Type *ResType = PointerType::getInt8PtrTy( V->getContext(), V->getType()->getPointerAddressSpace()); Constant *C = ConstantExpr::getPointerBitCastOrAddrSpaceCast(TransFun, ResType); Type *Int8PtrTyPrivate = Type::getInt8PtrTy(*Context, SPIRAS_Private); IntegerType *Int32Ty = Type::getInt32Ty(*Context); llvm::Constant *Fields[5] = { C, ConstantExpr::getBitCast(GS, Int8PtrTyPrivate), UndefValue::get(Int8PtrTyPrivate), UndefValue::get(Int32Ty), UndefValue::get(Int8PtrTyPrivate)}; GlobalAnnotations.push_back(ConstantStruct::getAnon(Fields)); } } void SPIRVToLLVM::transGlobalAnnotations() { if (!GlobalAnnotations.empty()) { Constant *Array = ConstantArray::get(ArrayType::get(GlobalAnnotations[0]->getType(), GlobalAnnotations.size()), GlobalAnnotations); auto *GV = new GlobalVariable(*M, Array->getType(), /*IsConstant*/ false, GlobalValue::AppendingLinkage, Array, "llvm.global.annotations"); GV->setSection("llvm.metadata"); } } static llvm::MDNode * transDecorationsToMetadataList(llvm::LLVMContext *Context, std::vector Decorates) { SmallVector MDs; MDs.reserve(Decorates.size()); for (const auto *Deco : Decorates) { std::vector OPs; auto *KindMD = ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Deco->getDecorateKind())); OPs.push_back(KindMD); switch (static_cast(Deco->getDecorateKind())) { case DecorationLinkageAttributes: { const auto *const LinkAttrDeco = static_cast(Deco); auto *const LinkNameMD = MDString::get(*Context, LinkAttrDeco->getLinkageName()); auto *const LinkTypeMD = ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(*Context), LinkAttrDeco->getLinkageType())); OPs.push_back(LinkNameMD); OPs.push_back(LinkTypeMD); break; } case spv::internal::DecorationHostAccessINTEL: case DecorationHostAccessINTEL: { const auto *const HostAccDeco = static_cast(Deco); auto *const AccModeMD = ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(*Context), HostAccDeco->getAccessMode())); auto *const NameMD = MDString::get(*Context, HostAccDeco->getVarName()); OPs.push_back(AccModeMD); OPs.push_back(NameMD); break; } case DecorationMergeINTEL: { const auto MergeAttrLits = Deco->getVecLiteral(); std::string FirstString = getString(MergeAttrLits); std::string SecondString = getString(MergeAttrLits.cbegin() + getVec(FirstString).size(), MergeAttrLits.cend()); OPs.push_back(MDString::get(*Context, FirstString)); OPs.push_back(MDString::get(*Context, SecondString)); break; } case DecorationMemoryINTEL: case DecorationUserSemantic: { auto *const StrMD = MDString::get(*Context, getString(Deco->getVecLiteral())); OPs.push_back(StrMD); break; } default: { for (const SPIRVWord Lit : Deco->getVecLiteral()) { auto *const LitMD = ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Lit)); OPs.push_back(LitMD); } break; } } MDs.push_back(MDNode::get(*Context, OPs)); } return MDNode::get(*Context, MDs); } void SPIRVToLLVM::transDecorationsToMetadata(SPIRVValue *BV, Value *V) { if (!BV->isVariable() && !BV->isInst()) return; auto SetDecorationsMetadata = [&](auto V) { std::vector Decorates = BV->getDecorations(); if (!Decorates.empty()) { MDNode *MDList = transDecorationsToMetadataList(Context, Decorates); V->setMetadata(SPIRV_MD_DECORATIONS, MDList); } }; if (auto *GV = dyn_cast(V)) SetDecorationsMetadata(GV); else if (auto *I = dyn_cast(V)) SetDecorationsMetadata(I); } namespace { static float convertSPIRVWordToFloat(SPIRVWord Spir) { union { float F; SPIRVWord Spir; } FPMaxError; FPMaxError.Spir = Spir; return FPMaxError.F; } static bool transFPMaxErrorDecoration(SPIRVValue *BV, Value *V, LLVMContext *Context) { SPIRVWord ID; if (Instruction *I = dyn_cast(V)) if (BV->hasDecorate(DecorationFPMaxErrorDecorationINTEL, 0, &ID)) { auto Literals = BV->getDecorationLiterals(DecorationFPMaxErrorDecorationINTEL); assert(Literals.size() == 1 && "FP Max Error decoration shall have 1 operand"); auto F = convertSPIRVWordToFloat(Literals[0]); if (CallInst *CI = dyn_cast(I)) { // Add attribute auto A = llvm::Attribute::get(*Context, "fpbuiltin-max-error", std::to_string(F)); CI->addFnAttr(A); } else { // Add metadata MDNode *N = MDNode::get(*Context, MDString::get(*Context, std::to_string(F))); I->setMetadata("fpbuiltin-max-error", N); } return true; } return false; } } // namespace bool SPIRVToLLVM::transDecoration(SPIRVValue *BV, Value *V) { if (transFPMaxErrorDecoration(BV, V, Context)) return true; if (!transAlign(BV, V)) return false; transIntelFPGADecorations(BV, V); transMemAliasingINTELDecorations(BV, V); // Decoration metadata is only enabled in SPIR-V friendly mode if (BM->getDesiredBIsRepresentation() == BIsRepresentation::SPIRVFriendlyIR) transDecorationsToMetadata(BV, V); DbgTran->transDbgInfo(BV, V); return true; } void SPIRVToLLVM::transGlobalCtorDtors(SPIRVVariable *BV) { if (BV->getName() != "llvm.global_ctors" && BV->getName() != "llvm.global_dtors") return; Value *V = transValue(BV, nullptr, nullptr); cast(V)->setLinkage(GlobalValue::AppendingLinkage); } void SPIRVToLLVM::createCXXStructor(const char *ListName, SmallVectorImpl &Funcs) { if (Funcs.empty()) return; // If the SPIR-V input contained a variable for the structor list and it // has already been translated, then don't interfere. if (M->getGlobalVariable(ListName)) return; // Type of a structor entry: { i32, void ()*, i8* } Type *PriorityTy = Type::getInt32Ty(*Context); PointerType *CtorTy = PointerType::getUnqual( FunctionType::get(Type::getVoidTy(*Context), false)); PointerType *ComdatTy = Type::getInt8PtrTy(*Context); StructType *StructorTy = StructType::get(PriorityTy, CtorTy, ComdatTy); ArrayType *ArrTy = ArrayType::get(StructorTy, Funcs.size()); GlobalVariable *GV = cast(M->getOrInsertGlobal(ListName, ArrTy)); GV->setLinkage(GlobalValue::AppendingLinkage); // Build the initializer. SmallVector ArrayElts; for (auto *F : Funcs) { SmallVector Elts; // SPIR-V does not specify an order between Initializers, so set default // priority. Elts.push_back(ConstantInt::get(PriorityTy, 65535)); Elts.push_back(ConstantExpr::getBitCast(F, CtorTy)); Elts.push_back(ConstantPointerNull::get(ComdatTy)); ArrayElts.push_back(ConstantStruct::get(StructorTy, Elts)); } Constant *NewArray = ConstantArray::get(ArrTy, ArrayElts); GV->setInitializer(NewArray); } bool SPIRVToLLVM::transFPContractMetadata() { bool ContractOff = false; for (unsigned I = 0, E = BM->getNumFunctions(); I != E; ++I) { SPIRVFunction *BF = BM->getFunction(I); if (!isKernel(BF)) continue; if (BF->getExecutionMode(ExecutionModeContractionOff)) { ContractOff = true; break; } } if (!ContractOff) M->getOrInsertNamedMetadata(kSPIR2MD::FPContract); return true; } std::string SPIRVToLLVM::transOCLImageTypeAccessQualifier(SPIRV::SPIRVTypeImage *ST) { return SPIRSPIRVAccessQualifierMap::rmap(ST->hasAccessQualifier() ? ST->getAccessQualifier() : AccessQualifierReadOnly); } bool SPIRVToLLVM::transNonTemporalMetadata(Instruction *I) { Constant *One = ConstantInt::get(Type::getInt32Ty(*Context), 1); MDNode *Node = MDNode::get(*Context, ConstantAsMetadata::get(One)); I->setMetadata(M->getMDKindID("nontemporal"), Node); return true; } // Information of types of kernel arguments may be additionally stored in // 'OpString "kernel_arg_type.%kernel_name%.type1,type2,type3,..' instruction. // Try to find such instruction and generate metadata based on it. // Return 'true' if 'OpString' was found and 'kernel_arg_type' metadata // generated and 'false' otherwise. static bool transKernelArgTypeMedataFromString(LLVMContext *Ctx, SPIRVModule *BM, Function *Kernel, std::string MDName) { // Run W/A translation only if the appropriate option is passed if (!BM->shouldPreserveOCLKernelArgTypeMetadataThroughString()) return false; std::string ArgTypePrefix = std::string(MDName) + "." + Kernel->getName().str() + "."; auto ArgTypeStrIt = std::find_if( BM->getStringVec().begin(), BM->getStringVec().end(), [=](SPIRVString *S) { return S->getStr().find(ArgTypePrefix) == 0; }); if (ArgTypeStrIt == BM->getStringVec().end()) return false; std::string ArgTypeStr = (*ArgTypeStrIt)->getStr().substr(ArgTypePrefix.size()); std::vector TypeMDs; int CountBraces = 0; std::string::size_type Start = 0; for (std::string::size_type I = 0; I < ArgTypeStr.length(); I++) { switch (ArgTypeStr[I]) { case '<': CountBraces++; break; case '>': CountBraces--; break; case ',': if (CountBraces == 0) { TypeMDs.push_back( MDString::get(*Ctx, ArgTypeStr.substr(Start, I - Start))); Start = I + 1; } } } Kernel->setMetadata(MDName, MDNode::get(*Ctx, TypeMDs)); return true; } void SPIRVToLLVM::transFunctionDecorationsToMetadata(SPIRVFunction *BF, Function *F) { size_t TotalParameterDecorations = 0; BF->foreachArgument([&](SPIRVFunctionParameter *Arg) { TotalParameterDecorations += Arg->getNumDecorations(); }); if (TotalParameterDecorations == 0) return; // Generate metadata for spirv.ParameterDecorations addKernelArgumentMetadata(Context, SPIRV_MD_PARAMETER_DECORATIONS, BF, F, [=](SPIRVFunctionParameter *Arg) { return transDecorationsToMetadataList( Context, Arg->getDecorations()); }); } bool SPIRVToLLVM::transMetadata() { SmallVector CtorKernels; for (unsigned I = 0, E = BM->getNumFunctions(); I != E; ++I) { SPIRVFunction *BF = BM->getFunction(I); Function *F = static_cast(getTranslatedValue(BF)); assert(F && "Invalid translated function"); transOCLMetadata(BF); transVectorComputeMetadata(BF); transFPGAFunctionMetadata(BF, F); // Decoration metadata is only enabled in SPIR-V friendly mode if (BM->getDesiredBIsRepresentation() == BIsRepresentation::SPIRVFriendlyIR) transFunctionDecorationsToMetadata(BF, F); if (F->getCallingConv() != CallingConv::SPIR_KERNEL) continue; // Generate metadata for reqd_work_group_size if (auto EM = BF->getExecutionMode(ExecutionModeLocalSize)) { F->setMetadata(kSPIR2MD::WGSize, getMDNodeStringIntVec(Context, EM->getLiterals())); } else if (auto *EM = BF->getExecutionModeId(ExecutionModeLocalSizeId)) { std::vector Values; for (const auto Id : EM->getLiterals()) { if (auto Val = transIdAsConstant(Id)) { Values.emplace_back(static_cast(*Val)); } } F->setMetadata(kSPIR2MD::WGSize, getMDNodeStringIntVec(Context, Values)); } // Generate metadata for work_group_size_hint if (auto EM = BF->getExecutionMode(ExecutionModeLocalSizeHint)) { F->setMetadata(kSPIR2MD::WGSizeHint, getMDNodeStringIntVec(Context, EM->getLiterals())); } else if (auto *EM = BF->getExecutionModeId(ExecutionModeLocalSizeHintId)) { std::vector Values; for (const auto Id : EM->getLiterals()) { if (auto Val = transIdAsConstant(Id)) { Values.emplace_back(static_cast(*Val)); } } F->setMetadata(kSPIR2MD::WGSizeHint, getMDNodeStringIntVec(Context, Values)); } // Generate metadata for vec_type_hint if (auto EM = BF->getExecutionMode(ExecutionModeVecTypeHint)) { std::vector MetadataVec; Type *VecHintTy = decodeVecTypeHint(*Context, EM->getLiterals()[0]); assert(VecHintTy); MetadataVec.push_back(ValueAsMetadata::get(UndefValue::get(VecHintTy))); MetadataVec.push_back(ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), 1))); F->setMetadata(kSPIR2MD::VecTyHint, MDNode::get(*Context, MetadataVec)); } // Generate metadata for Initializer. if (BF->getExecutionMode(ExecutionModeInitializer)) { CtorKernels.push_back(F); } // Generate metadata for intel_reqd_sub_group_size if (auto *EM = BF->getExecutionMode(ExecutionModeSubgroupSize)) { auto SizeMD = ConstantAsMetadata::get(getUInt32(M, EM->getLiterals()[0])); F->setMetadata(kSPIR2MD::SubgroupSize, MDNode::get(*Context, SizeMD)); } // Generate metadata for SubgroupsPerWorkgroup/SubgroupsPerWorkgroupId. auto EmitSubgroupsPerWorkgroupMD = [this, F](SPIRVExecutionModeKind EMK, uint64_t Value) { NamedMDNode *ExecModeMD = M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode); SmallVector OperandVec; OperandVec.push_back(ConstantAsMetadata::get(F)); OperandVec.push_back(ConstantAsMetadata::get(getUInt32(M, EMK))); OperandVec.push_back(ConstantAsMetadata::get(getUInt32(M, Value))); ExecModeMD->addOperand(MDNode::get(*Context, OperandVec)); }; if (auto *EM = BF->getExecutionMode(ExecutionModeSubgroupsPerWorkgroup)) { EmitSubgroupsPerWorkgroupMD(EM->getExecutionMode(), EM->getLiterals()[0]); } else if (auto *EM = BF->getExecutionModeId( ExecutionModeSubgroupsPerWorkgroupId)) { if (auto Val = transIdAsConstant(EM->getLiterals()[0])) { EmitSubgroupsPerWorkgroupMD(EM->getExecutionMode(), *Val); } } // Generate metadata for max_work_group_size if (auto EM = BF->getExecutionMode(ExecutionModeMaxWorkgroupSizeINTEL)) { F->setMetadata(kSPIR2MD::MaxWGSize, getMDNodeStringIntVec(Context, EM->getLiterals())); } // Generate metadata for no_global_work_offset if (BF->getExecutionMode(ExecutionModeNoGlobalOffsetINTEL)) { F->setMetadata(kSPIR2MD::NoGlobalOffset, MDNode::get(*Context, {})); } // Generate metadata for max_global_work_dim if (auto EM = BF->getExecutionMode(ExecutionModeMaxWorkDimINTEL)) { F->setMetadata(kSPIR2MD::MaxWGDim, getMDNodeStringIntVec(Context, EM->getLiterals())); } // Generate metadata for num_simd_work_items if (auto EM = BF->getExecutionMode(ExecutionModeNumSIMDWorkitemsINTEL)) { F->setMetadata(kSPIR2MD::NumSIMD, getMDNodeStringIntVec(Context, EM->getLiterals())); } // Generate metadata for scheduler_target_fmax_mhz if (auto EM = BF->getExecutionMode(ExecutionModeSchedulerTargetFmaxMhzINTEL)) { F->setMetadata(kSPIR2MD::FmaxMhz, getMDNodeStringIntVec(Context, EM->getLiterals())); } // Generate metadata for Intel FPGA streaming interface if (auto *EM = BF->getExecutionMode(ExecutionModeStreamingInterfaceINTEL)) { std::vector InterfaceVec = EM->getLiterals(); assert(InterfaceVec.size() == 1 && "Expected StreamingInterfaceINTEL to have exactly 1 literal"); std::vector InterfaceMDVec = [&]() -> std::vector { switch (InterfaceVec[0]) { case 0: return {MDString::get(*Context, "streaming")}; case 1: return {MDString::get(*Context, "streaming"), MDString::get(*Context, "stall_free_return")}; default: llvm_unreachable("Invalid streaming interface mode"); } }(); F->setMetadata(kSPIR2MD::IntelFPGAIPInterface, MDNode::get(*Context, InterfaceMDVec)); } if (auto *EM = BF->getExecutionMode(ExecutionModeMaximumRegistersINTEL)) { NamedMDNode *ExecModeMD = M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode); SmallVector ValueVec; ValueVec.push_back(ConstantAsMetadata::get(F)); ValueVec.push_back( ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode()))); ValueVec.push_back( ConstantAsMetadata::get(getUInt32(M, EM->getLiterals()[0]))); ExecModeMD->addOperand(MDNode::get(*Context, ValueVec)); } if (auto *EM = BF->getExecutionMode(ExecutionModeMaximumRegistersIdINTEL)) { NamedMDNode *ExecModeMD = M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode); SmallVector ValueVec; ValueVec.push_back(ConstantAsMetadata::get(F)); ValueVec.push_back( ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode()))); auto *ExecOp = BF->getModule()->getValue(EM->getLiterals()[0]); ValueVec.push_back( MDNode::get(*Context, ConstantAsMetadata::get(cast( transValue(ExecOp, nullptr, nullptr))))); ExecModeMD->addOperand(MDNode::get(*Context, ValueVec)); } if (auto *EM = BF->getExecutionMode(ExecutionModeNamedMaximumRegistersINTEL)) { NamedMDNode *ExecModeMD = M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode); SmallVector ValueVec; ValueVec.push_back(ConstantAsMetadata::get(F)); ValueVec.push_back( ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode()))); assert(EM->getLiterals()[0] == 0 && "Invalid named maximum number of registers"); ValueVec.push_back(MDString::get(*Context, "AutoINTEL")); ExecModeMD->addOperand(MDNode::get(*Context, ValueVec)); } } NamedMDNode *MemoryModelMD = M->getOrInsertNamedMetadata(kSPIRVMD::MemoryModel); MemoryModelMD->addOperand( getMDTwoInt(Context, static_cast(BM->getAddressingModel()), static_cast(BM->getMemoryModel()))); createCXXStructor("llvm.global_ctors", CtorKernels); return true; } bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) { Function *F = static_cast(getTranslatedValue(BF)); assert(F && "Invalid translated function"); if (F->getCallingConv() != CallingConv::SPIR_KERNEL) return true; if (BF->hasDecorate(DecorationVectorComputeFunctionINTEL)) return true; // Generate metadata for kernel_arg_addr_space addKernelArgumentMetadata( Context, SPIR_MD_KERNEL_ARG_ADDR_SPACE, BF, F, [=](SPIRVFunctionParameter *Arg) { SPIRVType *ArgTy = Arg->getType(); SPIRAddressSpace AS = SPIRAS_Private; if (ArgTy->isTypePointer()) AS = SPIRSPIRVAddrSpaceMap::rmap(ArgTy->getPointerStorageClass()); else if (ArgTy->isTypeOCLImage() || ArgTy->isTypePipe()) AS = SPIRAS_Global; return ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), AS)); }); // Generate metadata for kernel_arg_access_qual addKernelArgumentMetadata(Context, SPIR_MD_KERNEL_ARG_ACCESS_QUAL, BF, F, [=](SPIRVFunctionParameter *Arg) { std::string Qual; auto *T = Arg->getType(); if (T->isTypeOCLImage()) { auto *ST = static_cast(T); Qual = transOCLImageTypeAccessQualifier(ST); } else if (T->isTypePipe()) { auto *PT = static_cast(T); Qual = transOCLPipeTypeAccessQualifier(PT); } else Qual = "none"; return MDString::get(*Context, Qual); }); // Generate metadata for kernel_arg_type if (!transKernelArgTypeMedataFromString(Context, BM, F, SPIR_MD_KERNEL_ARG_TYPE)) addKernelArgumentMetadata(Context, SPIR_MD_KERNEL_ARG_TYPE, BF, F, [=](SPIRVFunctionParameter *Arg) { return transOCLKernelArgTypeName(Arg); }); // Generate metadata for kernel_arg_type_qual if (!transKernelArgTypeMedataFromString(Context, BM, F, SPIR_MD_KERNEL_ARG_TYPE_QUAL)) addKernelArgumentMetadata( Context, SPIR_MD_KERNEL_ARG_TYPE_QUAL, BF, F, [=](SPIRVFunctionParameter *Arg) { std::string Qual; if (Arg->hasDecorate(DecorationVolatile)) Qual = kOCLTypeQualifierName::Volatile; Arg->foreachAttr([&](SPIRVFuncParamAttrKind Kind) { Qual += Qual.empty() ? "" : " "; if (Kind == FunctionParameterAttributeNoAlias) Qual += kOCLTypeQualifierName::Restrict; }); if (Arg->getType()->isTypePipe()) { Qual += Qual.empty() ? "" : " "; Qual += kOCLTypeQualifierName::Pipe; } return MDString::get(*Context, Qual); }); // Generate metadata for kernel_arg_base_type addKernelArgumentMetadata(Context, SPIR_MD_KERNEL_ARG_BASE_TYPE, BF, F, [=](SPIRVFunctionParameter *Arg) { return transOCLKernelArgTypeName(Arg); }); // Generate metadata for kernel_arg_name if (BM->isGenArgNameMDEnabled()) { addKernelArgumentMetadata(Context, SPIR_MD_KERNEL_ARG_NAME, BF, F, [=](SPIRVFunctionParameter *Arg) { return MDString::get(*Context, Arg->getName()); }); } // Generate metadata for kernel_arg_buffer_location addBufferLocationMetadata(Context, BF, F, [=](SPIRVFunctionParameter *Arg) { auto Literals = Arg->getDecorationLiterals(DecorationBufferLocationINTEL); assert(Literals.size() == 1 && "BufferLocationINTEL decoration shall have 1 ID literal"); return ConstantAsMetadata::get( ConstantInt::get(Type::getInt32Ty(*Context), Literals[0])); }); // Generate metadata for kernel_arg_runtime_aligned addRuntimeAlignedMetadata(Context, BF, F, [=](SPIRVFunctionParameter *Arg) { return ConstantAsMetadata::get( ConstantInt::get(Type::getInt1Ty(*Context), 1)); }); return true; } bool SPIRVToLLVM::transVectorComputeMetadata(SPIRVFunction *BF) { using namespace VectorComputeUtil; Function *F = static_cast(getTranslatedValue(BF)); assert(F && "Invalid translated function"); if (BF->hasDecorate(DecorationStackCallINTEL)) F->addFnAttr(kVCMetadata::VCStackCall); if (BF->hasDecorate(DecorationVectorComputeFunctionINTEL)) F->addFnAttr(kVCMetadata::VCFunction); SPIRVWord SIMTMode = 0; if (BF->hasDecorate(DecorationSIMTCallINTEL, 0, &SIMTMode)) F->addFnAttr(kVCMetadata::VCSIMTCall, std::to_string(SIMTMode)); auto SEVAttr = translateSEVMetadata(BF, F->getContext()); if (SEVAttr) F->addAttributeAtIndex(AttributeList::ReturnIndex, SEVAttr.getValue()); for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { auto ArgNo = I->getArgNo(); SPIRVFunctionParameter *BA = BF->getArgument(ArgNo); SPIRVWord Kind; if (BA->hasDecorate(DecorationFuncParamIOKindINTEL, 0, &Kind)) { Attribute Attr = Attribute::get(*Context, kVCMetadata::VCArgumentIOKind, std::to_string(Kind)); F->addParamAttr(ArgNo, Attr); } SEVAttr = translateSEVMetadata(BA, F->getContext()); if (SEVAttr) F->addParamAttr(ArgNo, SEVAttr.getValue()); if (BA->hasDecorate(DecorationMediaBlockIOINTEL)) { assert(BA->getType()->isTypeImage() && "MediaBlockIOINTEL decoration is valid only on image parameters"); F->addParamAttr(ArgNo, Attribute::get(*Context, kVCMetadata::VCMediaBlockIO)); } } // Do not add float control if there is no any bool IsVCFloatControl = false; unsigned FloatControl = 0; // RoundMode and FloatMode are always same for all types in Cm // While Denorm could be different for double, float and half if (isKernel(BF)) { FPRoundingModeExecModeMap::foreach ( [&](FPRoundingMode VCRM, ExecutionMode EM) { if (BF->getExecutionMode(EM)) { IsVCFloatControl = true; FloatControl |= getVCFloatControl(VCRM); } }); FPOperationModeExecModeMap::foreach ( [&](FPOperationMode VCFM, ExecutionMode EM) { if (BF->getExecutionMode(EM)) { IsVCFloatControl = true; FloatControl |= getVCFloatControl(VCFM); } }); FPDenormModeExecModeMap::foreach ([&](FPDenormMode VCDM, ExecutionMode EM) { auto ExecModes = BF->getExecutionModeRange(EM); for (auto It = ExecModes.first; It != ExecModes.second; It++) { IsVCFloatControl = true; unsigned TargetWidth = (*It).second->getLiterals()[0]; VCFloatType FloatType = VCFloatTypeSizeMap::rmap(TargetWidth); FloatControl |= getVCFloatControl(VCDM, FloatType); } }); } else { if (BF->hasDecorate(DecorationFunctionRoundingModeINTEL)) { std::vector RoundModes = BF->getDecorations(DecorationFunctionRoundingModeINTEL); assert(RoundModes.size() == 3 && "Function must have precisely 3 " "FunctionRoundingModeINTEL decoration"); auto *DecRound = static_cast( RoundModes.at(0)); auto RoundingMode = DecRound->getRoundingMode(); #ifndef NDEBUG for (auto *DecPreCast : RoundModes) { auto *Dec = static_cast( DecPreCast); assert(Dec->getRoundingMode() == RoundingMode && "Rounding Mode must be equal within all targets"); } #endif IsVCFloatControl = true; FloatControl |= getVCFloatControl(RoundingMode); } if (BF->hasDecorate(DecorationFunctionDenormModeINTEL)) { std::vector DenormModes = BF->getDecorations(DecorationFunctionDenormModeINTEL); IsVCFloatControl = true; for (auto DecPtr : DenormModes) { auto DecDenorm = static_cast(DecPtr); VCFloatType FType = VCFloatTypeSizeMap::rmap(DecDenorm->getTargetWidth()); FloatControl |= getVCFloatControl(DecDenorm->getDenormMode(), FType); } } if (BF->hasDecorate(DecorationFunctionFloatingPointModeINTEL)) { std::vector FloatModes = BF->getDecorations(DecorationFunctionFloatingPointModeINTEL); assert(FloatModes.size() == 3 && "Function must have precisely 3 FunctionFloatingPointModeINTEL " "decoration"); auto *DecFlt = static_cast( FloatModes.at(0)); auto FloatingMode = DecFlt->getOperationMode(); #ifndef NDEBUG for (auto *DecPreCast : FloatModes) { auto *Dec = static_cast( DecPreCast); assert(Dec->getOperationMode() == FloatingMode && "Rounding Mode must be equal within all targets"); } #endif IsVCFloatControl = true; FloatControl |= getVCFloatControl(FloatingMode); } } if (IsVCFloatControl) { Attribute Attr = Attribute::get(*Context, kVCMetadata::VCFloatControl, std::to_string(FloatControl)); F->addFnAttr(Attr); } if (auto EM = BF->getExecutionMode(ExecutionModeSharedLocalMemorySizeINTEL)) { unsigned int SLMSize = EM->getLiterals()[0]; Attribute Attr = Attribute::get(*Context, kVCMetadata::VCSLMSize, std::to_string(SLMSize)); F->addFnAttr(Attr); } if (auto *EM = BF->getExecutionMode(ExecutionModeNamedBarrierCountINTEL)) { unsigned int NBarrierCnt = EM->getLiterals()[0]; Attribute Attr = Attribute::get(*Context, kVCMetadata::VCNamedBarrierCount, std::to_string(NBarrierCnt)); F->addFnAttr(Attr); } return true; } bool SPIRVToLLVM::transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F) { if (BF->hasDecorate(DecorationStallEnableINTEL)) { std::vector MetadataVec; MetadataVec.push_back(ConstantAsMetadata::get(getInt32(M, 1))); F->setMetadata(kSPIR2MD::StallEnable, MDNode::get(*Context, MetadataVec)); } if (BF->hasDecorate(DecorationFuseLoopsInFunctionINTEL)) { std::vector MetadataVec; auto Literals = BF->getDecorationLiterals(DecorationFuseLoopsInFunctionINTEL); MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[0]))); MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[1]))); F->setMetadata(kSPIR2MD::LoopFuse, MDNode::get(*Context, MetadataVec)); } if (BF->hasDecorate(internal::DecorationMathOpDSPModeINTEL)) { std::vector Literals = BF->getDecorationLiterals(internal::DecorationMathOpDSPModeINTEL); assert(Literals.size() == 2 && "MathOpDSPModeINTEL decoration shall have 2 literals"); F->setMetadata(kSPIR2MD::PreferDSP, MDNode::get(*Context, ConstantAsMetadata::get( getUInt32(M, Literals[0])))); if (Literals[1] != 0) { F->setMetadata(kSPIR2MD::PropDSPPref, MDNode::get(*Context, ConstantAsMetadata::get( getUInt32(M, Literals[1])))); } } if (BF->hasDecorate(internal::DecorationInitiationIntervalINTEL)) { std::vector MetadataVec; auto Literals = BF->getDecorationLiterals(internal::DecorationInitiationIntervalINTEL); MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[0]))); F->setMetadata(kSPIR2MD::InitiationInterval, MDNode::get(*Context, MetadataVec)); } if (BF->hasDecorate(internal::DecorationMaxConcurrencyINTEL)) { std::vector MetadataVec; auto Literals = BF->getDecorationLiterals(internal::DecorationMaxConcurrencyINTEL); MetadataVec.push_back(ConstantAsMetadata::get(getUInt32(M, Literals[0]))); F->setMetadata(kSPIR2MD::MaxConcurrency, MDNode::get(*Context, MetadataVec)); } if (BF->hasDecorate(internal::DecorationPipelineEnableINTEL)) { auto Literals = BF->getDecorationLiterals(internal::DecorationPipelineEnableINTEL); std::vector MetadataVec; MetadataVec.push_back(ConstantAsMetadata::get(getInt32(M, !Literals[0]))); F->setMetadata(kSPIR2MD::DisableLoopPipelining, MDNode::get(*Context, MetadataVec)); } return true; } bool SPIRVToLLVM::transAlign(SPIRVValue *BV, Value *V) { if (auto *AL = dyn_cast(V)) { if (auto Align = getAlignment(BV)) AL->setAlignment(llvm::Align(*Align)); return true; } if (auto *GV = dyn_cast(V)) { if (auto Align = getAlignment(BV)) GV->setAlignment(MaybeAlign(*Align)); return true; } return true; } Instruction *SPIRVToLLVM::transOCLBuiltinFromExtInst(SPIRVExtInst *BC, BasicBlock *BB) { assert(BB && "Invalid BB"); auto ExtOp = static_cast(BC->getExtOp()); std::string UnmangledName = OCLExtOpMap::map(ExtOp); assert(BM->getBuiltinSet(BC->getExtSetId()) == SPIRVEIS_OpenCL && "Not OpenCL extended instruction"); std::vector ArgTypes = transTypeVector(BC->getArgTypes()); std::vector PointerElementTys = getPointerElementTypes(BC->getArgTypes()); Type *RetTy = transType(BC->getType()); std::string MangledName = getSPIRVFriendlyIRFunctionName(ExtOp, ArgTypes, PointerElementTys, RetTy); SPIRVDBG(spvdbgs() << "[transOCLBuiltinFromExtInst] UnmangledName: " << UnmangledName << " MangledName: " << MangledName << '\n'); FunctionType *FT = FunctionType::get(RetTy, ArgTypes, false); Function *F = M->getFunction(MangledName); if (!F) { F = Function::Create(FT, GlobalValue::ExternalLinkage, MangledName, M); F->setCallingConv(CallingConv::SPIR_FUNC); if (isFuncNoUnwind()) F->addFnAttr(Attribute::NoUnwind); if (isFuncReadNone(UnmangledName)) F->setDoesNotAccessMemory(); } auto Args = transValue(BC->getArgValues(), F, BB); SPIRVDBG(dbgs() << "[transOCLBuiltinFromExtInst] Function: " << *F << ", Args: "; for (auto &I : Args) dbgs() << *I << ", "; dbgs() << '\n'); CallInst *CI = CallInst::Create(F, Args, BC->getName(), BB); setCallingConv(CI); addFnAttr(CI, Attribute::NoUnwind); return CI; } void SPIRVToLLVM::transAuxDataInst(SPIRVExtInst *BC) { assert(BC->getExtSetKind() == SPIRV::SPIRVEIS_NonSemantic_AuxData); if (!BC->getModule()->preserveAuxData()) return; auto Args = BC->getArguments(); // Args 0 and 1 are common between attributes and metadata. // 0 is the function, 1 is the name of the attribute/metadata as a string auto *SpvFcn = BC->getModule()->getValue(Args[0]); auto *F = static_cast(getTranslatedValue(SpvFcn)); assert(F && "Function should already have been translated!"); auto AttrOrMDName = BC->getModule()->get(Args[1])->getStr(); switch (BC->getExtOp()) { case NonSemanticAuxData::FunctionAttribute: { assert(Args.size() < 4 && "Unexpected FunctionAttribute Args"); // If this attr was specially handled and added elsewhere, skip it. const Attribute::AttrKind AsKind = Attribute::getAttrKindFromName(AttrOrMDName); if (AsKind != Attribute::None && F->hasFnAttribute(AsKind)) return; if (AsKind == Attribute::None && F->hasFnAttribute(AttrOrMDName)) return; // For attributes, arg 2 is the attribute value as a string, which may not // exist. if (Args.size() == 3) { auto AttrValue = BC->getModule()->get(Args[2])->getStr(); F->addFnAttr(AttrOrMDName, AttrValue); } else { if (AsKind != Attribute::None) F->addFnAttr(AsKind); else F->addFnAttr(AttrOrMDName); } break; } case NonSemanticAuxData::FunctionMetadata: { // If this metadata was specially handled and added elsewhere, skip it. if (F->hasMetadata(AttrOrMDName)) return; SmallVector MetadataArgs; // Process the metadata values. for (size_t CurArg = 2; CurArg < Args.size(); CurArg++) { auto *Arg = BC->getModule()->get(Args[CurArg]); // For metadata, the metadata values can be either values or strings. if (Arg->getOpCode() == OpString) { auto *ArgAsStr = static_cast(Arg); MetadataArgs.push_back( MDString::get(F->getContext(), ArgAsStr->getStr())); } else { auto *ArgAsVal = static_cast(Arg); auto *TranslatedMD = transValue(ArgAsVal, F, nullptr); MetadataArgs.push_back(ValueAsMetadata::get(TranslatedMD)); } } F->setMetadata(AttrOrMDName, MDNode::get(*Context, MetadataArgs)); break; } default: llvm_unreachable("Invalid op"); } } // SPIR-V only contains language version. Use OpenCL language version as // SPIR version. void SPIRVToLLVM::transSourceLanguage() { SPIRVWord Ver = 0; SourceLanguage Lang = BM->getSourceLanguage(&Ver); if (Lang != SourceLanguageUnknown && // Allow unknown for debug info test Lang != SourceLanguageOpenCL_C && Lang != SourceLanguageOpenCL_CPP) return; unsigned short Major = 0; unsigned char Minor = 0; unsigned char Rev = 0; std::tie(Major, Minor, Rev) = decodeOCLVer(Ver); SPIRVMDBuilder Builder(*M); Builder.addNamedMD(kSPIRVMD::Source).addOp().add(Lang).add(Ver).done(); // ToDo: Phasing out usage of old SPIR metadata if (Ver <= kOCLVer::CL12) addOCLVersionMetadata(Context, M, kSPIR2MD::SPIRVer, 1, 2); else addOCLVersionMetadata(Context, M, kSPIR2MD::SPIRVer, 2, 0); addOCLVersionMetadata(Context, M, kSPIR2MD::OCLVer, Major, Minor); } bool SPIRVToLLVM::transSourceExtension() { auto ExtSet = rmap(BM->getExtension()); auto CapSet = rmap(BM->getCapability()); ExtSet.insert(CapSet.begin(), CapSet.end()); auto OCLExtensions = map(ExtSet); std::set OCLOptionalCoreFeatures; static const char *OCLOptCoreFeatureNames[] = { "cl_images", "cl_doubles", }; for (auto &I : OCLOptCoreFeatureNames) { auto Loc = OCLExtensions.find(I); if (Loc != OCLExtensions.end()) { OCLExtensions.erase(Loc); OCLOptionalCoreFeatures.insert(I); } } addNamedMetadataStringSet(Context, M, kSPIR2MD::Extensions, OCLExtensions); addNamedMetadataStringSet(Context, M, kSPIR2MD::OptFeatures, OCLOptionalCoreFeatures); return true; } llvm::GlobalValue::LinkageTypes SPIRVToLLVM::transLinkageType(const SPIRVValue *V) { std::string ValueName = V->getName(); if (ValueName == "llvm.used" || ValueName == "llvm.compiler.used") return GlobalValue::AppendingLinkage; int LT = V->getLinkageType(); switch (LT) { case internal::LinkageTypeInternal: return GlobalValue::InternalLinkage; case LinkageTypeImport: // Function declaration if (V->getOpCode() == OpFunction) { if (static_cast(V)->getNumBasicBlock() == 0) return GlobalValue::ExternalLinkage; } // Variable declaration if (V->getOpCode() == OpVariable) { if (static_cast(V)->getInitializer() == 0) return GlobalValue::ExternalLinkage; } // Definition return GlobalValue::AvailableExternallyLinkage; case LinkageTypeExport: if (V->getOpCode() == OpVariable) { if (static_cast(V)->getInitializer() == 0) // Tentative definition return GlobalValue::CommonLinkage; } return GlobalValue::ExternalLinkage; case LinkageTypeLinkOnceODR: return GlobalValue::LinkOnceODRLinkage; default: llvm_unreachable("Invalid linkage type"); } } Instruction *SPIRVToLLVM::transAllAny(SPIRVInstruction *I, BasicBlock *BB) { CallInst *CI = cast(transSPIRVBuiltinFromInst(I, BB)); assert(CI->getCalledFunction() && "Unexpected indirect call"); BuiltinFuncMangleInfo BtnInfo; AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return cast(mapValue( I, mutateCallInst( M, CI, [=](CallInst *, std::vector &Args) { auto *OldArg = CI->getOperand(0); auto *NewArgTy = FixedVectorType::get( Type::getInt8Ty(*Context), cast(OldArg->getType())->getNumElements()); auto *NewArg = CastInst::CreateSExtOrBitCast(OldArg, NewArgTy, "", CI); Args[0] = NewArg; return getSPIRVFuncName(I->getOpCode(), getSPIRVFuncSuffix(I)); }, &BtnInfo, &Attrs, /*TakeFuncName=*/true))); } Instruction *SPIRVToLLVM::transRelational(SPIRVInstruction *I, BasicBlock *BB) { CallInst *CI = cast(transSPIRVBuiltinFromInst(I, BB)); assert(CI->getCalledFunction() && "Unexpected indirect call"); BuiltinFuncMangleInfo BtnInfo; AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return cast(mapValue( I, mutateCallInst( M, CI, [=](CallInst *, std::vector &Args, llvm::Type *&RetTy) { if (CI->getType()->isVectorTy()) { RetTy = FixedVectorType::get( Type::getInt8Ty(*Context), cast(CI->getType())->getNumElements()); } return getSPIRVFuncName(I->getOpCode(), getSPIRVFuncSuffix(I)); }, [=](CallInst *NewCI) -> Instruction * { Type *RetTy = CI->getType(); if (RetTy == NewCI->getType()) return NewCI; return CastInst::CreateTruncOrBitCast(NewCI, RetTy, "", NewCI->getNextNode()); }, &BtnInfo, &Attrs, /*TakeFuncName=*/true))); } llvm::Optional getSpirvReport(std::istream &IS) { int IgnoreErrCode; return getSpirvReport(IS, IgnoreErrCode); } llvm::Optional getSpirvReport(std::istream &IS, int &ErrCode) { SPIRVWord Word; std::string Name; std::unique_ptr BM(SPIRVModule::createSPIRVModule()); SPIRVDecoder D(IS, *BM); D >> Word; if (Word != MagicNumber) { ErrCode = SPIRVEC_InvalidMagicNumber; return {}; } D >> Word; if (!isSPIRVVersionKnown(Word)) { ErrCode = SPIRVEC_InvalidVersionNumber; return {}; } SPIRVModuleReport Report; Report.Version = static_cast(Word); // Skip: Generator’s magic number, Bound and Reserved word D.ignore(3); bool IsReportGenCompleted = false, IsMemoryModelDefined = false; while (!IS.bad() && !IsReportGenCompleted && D.getWordCountAndOpCode()) { switch (D.OpCode) { case OpCapability: D >> Word; Report.Capabilities.push_back(Word); break; case OpExtension: Name.clear(); D >> Name; Report.Extensions.push_back(Name); break; case OpExtInstImport: Name.clear(); D >> Word >> Name; Report.ExtendedInstructionSets.push_back(Name); break; case OpMemoryModel: if (IsMemoryModelDefined) { ErrCode = SPIRVEC_RepeatedMemoryModel; return {}; } SPIRVAddressingModelKind AddrModel; SPIRVMemoryModelKind MemoryModel; D >> AddrModel >> MemoryModel; if (!isValid(AddrModel)) { ErrCode = SPIRVEC_InvalidAddressingModel; return {}; } if (!isValid(MemoryModel)) { ErrCode = SPIRVEC_InvalidMemoryModel; return {}; } Report.MemoryModel = MemoryModel; Report.AddrModel = AddrModel; IsMemoryModelDefined = true; // In this report we don't analyze instructions after OpMemoryModel IsReportGenCompleted = true; break; default: // No more instructions to gather information about IsReportGenCompleted = true; } } if (IS.bad()) { ErrCode = SPIRVEC_InvalidModule; return {}; } if (!IsMemoryModelDefined) { ErrCode = SPIRVEC_UnspecifiedMemoryModel; return {}; } ErrCode = SPIRVEC_Success; return llvm::Optional(std::move(Report)); } std::string formatAddressingModel(uint32_t AddrModel) { switch (AddrModel) { case AddressingModelLogical: return "Logical"; case AddressingModelPhysical32: return "Physical32"; case AddressingModelPhysical64: return "Physical64"; case AddressingModelPhysicalStorageBuffer64: return "PhysicalStorageBuffer64"; default: return "Unknown"; } } std::string formatMemoryModel(uint32_t MemoryModel) { switch (MemoryModel) { case MemoryModelSimple: return "Simple"; case MemoryModelGLSL450: return "GLSL450"; case MemoryModelOpenCL: return "OpenCL"; case MemoryModelVulkan: return "Vulkan"; default: return "Unknown"; } } SPIRVModuleTextReport formatSpirvReport(const SPIRVModuleReport &Report) { SPIRVModuleTextReport TextReport; TextReport.Version = formatVersionNumber(static_cast(Report.Version)); TextReport.AddrModel = formatAddressingModel(Report.AddrModel); TextReport.MemoryModel = formatMemoryModel(Report.MemoryModel); // format capability codes as strings std::string Name; for (auto Capability : Report.Capabilities) { const bool Found = SPIRVCapabilityNameMap::find( static_cast(Capability), &Name); TextReport.Capabilities.push_back(Found ? Name : "Unknown"); } // other fields with string content can be copied as is TextReport.Extensions = Report.Extensions; TextReport.ExtendedInstructionSets = Report.ExtendedInstructionSets; return TextReport; } std::unique_ptr readSpirvModule(std::istream &IS, const SPIRV::TranslatorOpts &Opts, std::string &ErrMsg) { std::unique_ptr BM(SPIRVModule::createSPIRVModule(Opts)); IS >> *BM; if (!BM->isModuleValid()) { BM->getError(ErrMsg); return nullptr; } return BM; } std::unique_ptr readSpirvModule(std::istream &IS, std::string &ErrMsg) { SPIRV::TranslatorOpts DefaultOpts; return readSpirvModule(IS, DefaultOpts, ErrMsg); } } // namespace SPIRV std::unique_ptr llvm::convertSpirvToLLVM(LLVMContext &C, SPIRVModule &BM, const SPIRV::TranslatorOpts &Opts, std::string &ErrMsg) { std::unique_ptr M(new Module("", C)); SPIRVToLLVM BTL(M.get(), &BM); if (!BTL.translate()) { BM.getError(ErrMsg); return nullptr; } llvm::ModulePassManager PassMgr; addSPIRVBIsLoweringPass(PassMgr, Opts.getDesiredBIsRepresentation()); llvm::ModuleAnalysisManager MAM; MAM.registerPass([&] { return PassInstrumentationAnalysis(); }); PassMgr.run(*M, MAM); return M; } std::unique_ptr llvm::convertSpirvToLLVM(LLVMContext &C, SPIRVModule &BM, std::string &ErrMsg) { SPIRV::TranslatorOpts DefaultOpts; return llvm::convertSpirvToLLVM(C, BM, DefaultOpts, ErrMsg); } bool llvm::readSpirv(LLVMContext &C, std::istream &IS, Module *&M, std::string &ErrMsg) { SPIRV::TranslatorOpts DefaultOpts; // As it is stated in the documentation, the translator accepts all SPIR-V // extensions by default DefaultOpts.enableAllExtensions(); return llvm::readSpirv(C, DefaultOpts, IS, M, ErrMsg); } bool llvm::readSpirv(LLVMContext &C, const SPIRV::TranslatorOpts &Opts, std::istream &IS, Module *&M, std::string &ErrMsg) { std::unique_ptr BM(readSpirvModule(IS, Opts, ErrMsg)); if (!BM) return false; if (Opts.getFnVarSpecEnable()) { if (!specializeFnVariants(BM.get(), ErrMsg)) { return false; } // Write out the specialized/targeted module if (!BM->getFnVarSpvOut().empty()) { auto SaveOpt = SPIRVUseTextFormat; auto OFSSpv = std::ofstream(BM->getFnVarSpvOut(), std::ios::binary); SPIRVUseTextFormat = false; OFSSpv << *BM; if (BM->getError(ErrMsg) != SPIRVEC_Success) { return false; } SPIRVUseTextFormat = SaveOpt; } } if (BM->getExtension().find("SPV_INTEL_function_variants") != BM->getExtension().end()) { ErrMsg = "Instructions from SPV_INTEL_function_variants are not " "convertible to LLVM IR."; return false; } M = convertSpirvToLLVM(C, *BM, Opts, ErrMsg).release(); if (!M) return false; if (DbgSaveTmpLLVM) dumpLLVM(M, DbgTmpLLVMFileName); return true; } bool llvm::getSpecConstInfo(std::istream &IS, std::vector &SpecConstInfo) { std::unique_ptr BM(SPIRVModule::createSPIRVModule()); BM->setAutoAddExtensions(false); SPIRVDecoder D(IS, *BM); SPIRVWord Magic; D >> Magic; if (!BM->getErrorLog().checkError(Magic == MagicNumber, SPIRVEC_InvalidModule, "invalid magic number")) { return false; } // Skip the rest of the header D.ignore(4); // According to the logical layout of SPIRV module (p2.4 of the spec), // all constant instructions must appear before function declarations. while (D.OpCode != OpFunction && D.getWordCountAndOpCode()) { switch (D.OpCode) { case OpDecorate: // The decoration is added to the module in scope of SPIRVDecorate::decode D.getEntry(); break; case OpTypeBool: case OpTypeInt: case OpTypeFloat: BM->addEntry(D.getEntry()); break; case OpSpecConstant: case OpSpecConstantTrue: case OpSpecConstantFalse: { auto *C = BM->addConstant(static_cast(D.getEntry())); SPIRVWord SpecConstIdLiteral = 0; if (C->hasDecorate(DecorationSpecId, 0, &SpecConstIdLiteral)) { SPIRVType *Ty = C->getType(); uint32_t SpecConstSize = Ty->isTypeBool() ? 1 : Ty->getBitWidth() / 8; SpecConstInfo.emplace_back(SpecConstIdLiteral, SpecConstSize); } break; } default: D.ignoreInstruction(); } } return !IS.bad(); } // clang-format off const StringSet<> SPIRVToLLVM::BuiltInConstFunc { "convert", "get_work_dim", "get_global_size", "sub_group_ballot_bit_count", "get_global_id", "get_local_size", "get_local_id", "get_num_groups", "get_group_id", "get_global_offset", "acos", "acosh", "acospi", "asin", "asinh", "asinpi", "atan", "atan2", "atanh", "atanpi", "atan2pi", "cbrt", "ceil", "copysign", "cos", "cosh", "cospi", "erfc", "erf", "exp", "exp2", "exp10", "expm1", "fabs", "fdim", "floor", "fma", "fmax", "fmin", "fmod", "ilogb", "ldexp", "lgamma", "log", "log2", "log10", "log1p", "logb", "mad", "maxmag", "minmag", "nan", "nextafter", "pow", "pown", "powr", "remainder", "rint", "rootn", "round", "rsqrt", "sin", "sinh", "sinpi", "sqrt", "tan", "tanh", "tanpi", "tgamma", "trunc", "half_cos", "half_divide", "half_exp", "half_exp2", "half_exp10", "half_log", "half_log2", "half_log10", "half_powr", "half_recip", "half_rsqrt", "half_sin", "half_sqrt", "half_tan", "native_cos", "native_divide", "native_exp", "native_exp2", "native_exp10", "native_log", "native_log2", "native_log10", "native_powr", "native_recip", "native_rsqrt", "native_sin", "native_sqrt", "native_tan", "abs", "abs_diff", "add_sat", "hadd", "rhadd", "clamp", "clz", "mad_hi", "mad_sat", "max", "min", "mul_hi", "rotate", "sub_sat", "upsample", "popcount", "mad24", "mul24", "degrees", "mix", "radians", "step", "smoothstep", "sign", "cross", "dot", "distance", "length", "normalize", "fast_distance", "fast_length", "fast_normalize", "isequal", "isnotequal", "isgreater", "isgreaterequal", "isless", "islessequal", "islessgreater", "isfinite", "isinf", "isnan", "isnormal", "isordered", "isunordered", "signbit", "any", "all", "bitselect", "select", "shuffle", "shuffle2", "get_image_width", "get_image_height", "get_image_depth", "get_image_channel_data_type", "get_image_channel_order", "get_image_dim", "get_image_array_size", "get_image_array_size", "sub_group_inverse_ballot", "sub_group_ballot_bit_extract", }; // clang-format on SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVReader.h000066400000000000000000000274041506646577400213720ustar00rootroot00000000000000//===- SPIRVReader.h - Converts SPIR-V to LLVM ------------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file contains declaration of SPIRVToLLVM class which implements /// conversion of SPIR-V binary to LLVM IR. /// //===----------------------------------------------------------------------===// #ifndef SPIRVREADER_H #define SPIRVREADER_H #include "SPIRVInternal.h" #include "SPIRVModule.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/GlobalValue.h" // llvm::GlobalValue::LinkageTypes namespace llvm { class Metadata; class Module; class Type; class Instruction; class CallInst; class BasicBlock; class Loop; class Function; class GlobalVariable; class LLVMContext; class MDString; class IntrinsicInst; class LoadInst; class BranchInst; class BinaryOperator; class Value; } // namespace llvm using namespace llvm; namespace SPIRV { class SPIRVFunctionParameter; class SPIRVConstantSampler; class SPIRVConstantPipeStorage; class SPIRVLoopMerge; class SPIRVToLLVMDbgTran; class SPIRVToLLVM { public: SPIRVToLLVM(Module *LLVMModule, SPIRVModule *TheSPIRVModule); static const StringSet<> BuiltInConstFunc; Type *transType(SPIRVType *BT, bool IsClassMember = false); std::string transTypeToOCLTypeName(SPIRVType *BT, bool IsSigned = true); std::vector transTypeVector(const std::vector &); std::vector getPointerElementTypes(llvm::ArrayRef Tys); bool translate(); bool transAddressingModel(); Value *transValue(SPIRVValue *, Function *F, BasicBlock *, bool CreatePlaceHolder = true); Value *transValueWithoutDecoration(SPIRVValue *, Function *F, BasicBlock *, bool CreatePlaceHolder = true); bool transDecoration(SPIRVValue *, Value *); bool transAlign(SPIRVValue *, Value *); Instruction *transOCLBuiltinFromExtInst(SPIRVExtInst *BC, BasicBlock *BB); void transAuxDataInst(SPIRVExtInst *BC); std::vector transValue(const std::vector &, Function *F, BasicBlock *); Function *transFunction(SPIRVFunction *F, unsigned AS = SPIRAS_Private); void transFunctionAttrs(SPIRVFunction *BF, Function *F); Value *transBlockInvoke(SPIRVValue *Invoke, BasicBlock *BB); Instruction *transWGSizeQueryBI(SPIRVInstruction *BI, BasicBlock *BB); Instruction *transSGSizeQueryBI(SPIRVInstruction *BI, BasicBlock *BB); bool transFPContractMetadata(); bool transMetadata(); bool transOCLMetadata(SPIRVFunction *BF); bool transVectorComputeMetadata(SPIRVFunction *BF); bool transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F); Value *transAsmINTEL(SPIRVAsmINTEL *BA); CallInst *transAsmCallINTEL(SPIRVAsmCallINTEL *BI, Function *F, BasicBlock *BB); Value *transFixedPointInst(SPIRVInstruction *BI, BasicBlock *BB); Value *transArbFloatInst(SPIRVInstruction *BI, BasicBlock *BB, bool IsBinaryInst = false); bool transNonTemporalMetadata(Instruction *I); template void transAliasingMemAccess(SPIRVInstType *BI, Instruction *I); void addMemAliasMetadata(Instruction *I, SPIRVId AliasListId, uint32_t AliasMDKind); void transSourceLanguage(); bool transSourceExtension(); void transGeneratorMD(); Value *transConvertInst(SPIRVValue *BV, Function *F, BasicBlock *BB); Instruction *transBuiltinFromInst(const std::string &FuncName, SPIRVInstruction *BI, BasicBlock *BB); Instruction *transSPIRVBuiltinFromInst(SPIRVInstruction *BI, BasicBlock *BB); /// \brief Expand OCL builtin functions with scalar argument, e.g. /// step, smoothstep. /// gentype func (fp edge, gentype x) /// => /// gentype func (gentype edge, gentype x) /// \return transformed call instruction. CallInst *expandOCLBuiltinWithScalarArg(CallInst *CI, const std::string &FuncName); typedef DenseMap SPIRVToLLVMTypeMap; typedef DenseMap SPIRVToLLVMValueMap; typedef DenseMap SPIRVBlockToLLVMStructMap; typedef DenseMap SPIRVToLLVMFunctionMap; typedef DenseMap BuiltinVarMap; typedef std::unordered_map SPIRVToLLVMMDAliasInstMap; // A SPIRV value may be translated to a load instruction of a placeholder // global variable. This map records load instruction of these placeholders // which are supposed to be replaced by the real values later. typedef std::map SPIRVToLLVMPlaceholderMap; typedef std::map SPIRVToLLVMLoopMetadataMap; private: Module *M; LLVMContext *Context; SPIRVModule *BM; SPIRVToLLVMTypeMap TypeMap; SPIRVToLLVMValueMap ValueMap; SPIRVToLLVMFunctionMap FuncMap; SPIRVBlockToLLVMStructMap BlockMap; SPIRVToLLVMPlaceholderMap PlaceholderMap; std::unique_ptr DbgTran; // GlobalAnnotations collects array of annotation entries for global variables // and functions. They are used in translation of llvm.global.annotations // instruction. std::vector GlobalAnnotations; // AnnotationsMap helps to translate annotation strings for local variables. // Map values are pointers on global strings in LLVM-IR. It is used to avoid // duplication of these annotation strings in LLVM-IR, which can be caused by // multiple translation of UserSemantic decorations with the same literal. std::unordered_map AnnotationsMap; // Loops metadata is translated in the end of a function translation. // This storage contains pairs of translated loop header basic block and loop // metadata SPIR-V instruction in SPIR-V representation of this basic block. SPIRVToLLVMLoopMetadataMap FuncLoopMetadataMap; // These storages are used to prevent duplication of alias.scope/noalias // metadata SPIRVToLLVMMDAliasInstMap MDAliasDomainMap; SPIRVToLLVMMDAliasInstMap MDAliasScopeMap; SPIRVToLLVMMDAliasInstMap MDAliasListMap; Type *mapType(SPIRVType *BT, Type *T); // If a value is mapped twice, the existing mapped value is a placeholder, // which must be a load instruction of a global variable whose name starts // with kPlaceholderPrefix. Value *mapValue(SPIRVValue *BV, Value *V); // OpenCL function always has NoUnwind attribute. // Change this if it is no longer true. bool isFuncNoUnwind() const { return true; } bool isFuncReadNone(const std::string &Name) const { return BuiltInConstFunc.count(Name); } bool isDirectlyTranslatedToOCL(Op OpCode) const; MDString *transOCLKernelArgTypeName(SPIRVFunctionParameter *); // Attempt to translate Id as a (specialization) constant. llvm::Optional transIdAsConstant(SPIRVId Id); // Return the value of an Alignment or AlignmentId decoration for V. llvm::Optional getAlignment(SPIRVValue *V); Value *mapFunction(SPIRVFunction *BF, Function *F); Value *getTranslatedValue(SPIRVValue *BV); IntrinsicInst *getLifetimeStartIntrinsic(Instruction *I); SPIRVErrorLog &getErrorLog(); void setCallingConv(CallInst *Call); Type *transFPType(SPIRVType *T); Value *transShiftLogicalBitwiseInst(SPIRVValue *BV, BasicBlock *BB, Function *F); Value *transCmpInst(SPIRVValue *BV, BasicBlock *BB, Function *F); void transOCLBuiltinFromInstPreproc(SPIRVInstruction *BI, Type *&RetTy, std::vector &Args); Instruction *transOCLBuiltinPostproc(SPIRVInstruction *BI, CallInst *CI, BasicBlock *BB, const std::string &DemangledName); std::string transOCLImageTypeName(SPIRV::SPIRVTypeImage *ST); std::string transOCLSampledImageTypeName(SPIRV::SPIRVTypeSampledImage *ST); std::string transVMEImageTypeName(SPIRV::SPIRVTypeVmeImageINTEL *VT); std::string transPipeTypeName(SPIRV::SPIRVTypePipe *ST); std::string transOCLPipeStorageTypeName(SPIRV::SPIRVTypePipeStorage *PST); std::string transOCLImageTypeAccessQualifier(SPIRV::SPIRVTypeImage *ST); std::string transOCLPipeTypeAccessQualifier(SPIRV::SPIRVTypePipe *ST); std::string transVCTypeName(SPIRVTypeBufferSurfaceINTEL *PST); Value *oclTransConstantSampler(SPIRV::SPIRVConstantSampler *BCS, BasicBlock *BB); Value *oclTransConstantPipeStorage(SPIRV::SPIRVConstantPipeStorage *BCPS); void setName(llvm::Value *V, SPIRVValue *BV); template void setLLVMLoopMetadata(const LoopInstType *LM, const Loop *LoopObj); void transLLVMLoopMetadata(const Function *F); inline llvm::Metadata *getMetadataFromName(std::string Name); inline std::vector getMetadataFromNameAndParameter(std::string Name, SPIRVWord Parameter); inline MDNode *getMetadataFromNameAndParameter(std::string Name, int64_t Parameter); template bool foreachFuncCtlMask(Source, Func); llvm::GlobalValue::LinkageTypes transLinkageType(const SPIRVValue *V); Instruction *transAllAny(SPIRVInstruction *BI, BasicBlock *BB); Instruction *transRelational(SPIRVInstruction *BI, BasicBlock *BB); void transUserSemantic(SPIRV::SPIRVFunction *Fun); void transGlobalAnnotations(); void transGlobalCtorDtors(SPIRVVariable *BV); void createCXXStructor(const char *ListName, SmallVectorImpl &Funcs); void transIntelFPGADecorations(SPIRVValue *BV, Value *V); void transMemAliasingINTELDecorations(SPIRVValue *BV, Value *V); void transDecorationsToMetadata(SPIRVValue *BV, Value *V); void transFunctionDecorationsToMetadata(SPIRVFunction *BF, Function *F); void transFunctionPointerCallArgumentAttributes(SPIRVValue *BV, CallInst *CI, SPIRVTypeFunction *CalledFnTy); }; // class SPIRVToLLVM } // namespace SPIRV #endif // SPIRVREADER_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVRegularizeLLVM.cpp000066400000000000000000000567231506646577400233350ustar00rootroot00000000000000//===- SPIRVRegularizeLLVM.cpp - Regularize LLVM for SPIR-V ------- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements regularization of LLVM module for SPIR-V. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spvregular" #include "SPIRVRegularizeLLVM.h" #include "OCLUtil.h" #include "SPIRVInternal.h" #include "SPIRVMDWalker.h" #include "libSPIRV/SPIRVDebug.h" #include "llvm/ADT/StringExtras.h" // llvm::isDigit #include "llvm/CodeGen/IntrinsicLowering.h" #include "llvm/Demangle/Demangle.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/Support/Debug.h" #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" // expandMemSetAsLoop() #include #include using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { static bool SPIRVDbgSaveRegularizedModule = false; static std::string RegularizedModuleTmpFile = "regularized.bc"; char SPIRVRegularizeLLVMLegacy::ID = 0; bool SPIRVRegularizeLLVMLegacy::runOnModule(Module &Module) { return runRegularizeLLVM(Module); } std::string SPIRVRegularizeLLVMBase::lowerLLVMIntrinsicName(IntrinsicInst *II) { Function *IntrinsicFunc = II->getCalledFunction(); assert(IntrinsicFunc && "Missing function"); std::string FuncName = IntrinsicFunc->getName().str(); std::replace(FuncName.begin(), FuncName.end(), '.', '_'); FuncName = "spirv." + FuncName; return FuncName; } void SPIRVRegularizeLLVMBase::lowerIntrinsicToFunction( IntrinsicInst *Intrinsic) { // For @llvm.memset.* intrinsic cases with constant value and length arguments // are emulated via "storing" a constant array to the destination. For other // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the // intrinsic to a loop via expandMemSetAsLoop() from // llvm/Transforms/Utils/LowerMemIntrinsics.h if (auto *MSI = dyn_cast(Intrinsic)) if (isa(MSI->getValue()) && isa(MSI->getLength())) return; // To be handled in LLVMToSPIRV::transIntrinsicInst std::string FuncName = lowerLLVMIntrinsicName(Intrinsic); if (Intrinsic->isVolatile()) FuncName += ".volatile"; // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_* Function *F = M->getFunction(FuncName); if (F) { // This function is already linked in. Intrinsic->setCalledFunction(F); return; } // TODO copy arguments attributes: nocapture writeonly. FunctionCallee FC = M->getOrInsertFunction(FuncName, Intrinsic->getFunctionType()); auto IntrinsicID = Intrinsic->getIntrinsicID(); Intrinsic->setCalledFunction(FC); F = dyn_cast(FC.getCallee()); assert(F && "must be a function!"); switch (IntrinsicID) { case Intrinsic::memset: { auto *MSI = static_cast(Intrinsic); Argument *Dest = F->getArg(0); Argument *Val = F->getArg(1); Argument *Len = F->getArg(2); Argument *IsVolatile = F->getArg(3); Dest->setName("dest"); Val->setName("val"); Len->setName("len"); IsVolatile->setName("isvolatile"); IsVolatile->addAttr(Attribute::ImmArg); BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F); IRBuilder<> IRB(EntryBB); auto *MemSet = IRB.CreateMemSet(Dest, Val, Len, MSI->getDestAlign(), MSI->isVolatile()); IRB.CreateRetVoid(); expandMemSetAsLoop(cast(MemSet)); MemSet->eraseFromParent(); break; } case Intrinsic::bswap: { BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F); IRBuilder<> IRB(EntryBB); auto *BSwap = IRB.CreateIntrinsic(Intrinsic::bswap, Intrinsic->getType(), F->getArg(0)); IRB.CreateRet(BSwap); IntrinsicLowering IL(M->getDataLayout()); IL.LowerIntrinsicCall(BSwap); break; } default: break; // do nothing } return; } void SPIRVRegularizeLLVMBase::lowerFunnelShift(IntrinsicInst *FSHIntrinsic) { // Get a separate function - otherwise, we'd have to rework the CFG of the // current one. Then simply replace the intrinsic uses with a call to the new // function. // Expected LLVM IR for the function: i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* // %c) FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType(); Type *FSHRetTy = FSHFuncTy->getReturnType(); const std::string FuncName = lowerLLVMIntrinsicName(FSHIntrinsic); Function *FSHFunc = getOrCreateFunction(M, FSHRetTy, FSHFuncTy->params(), FuncName); if (!FSHFunc->empty()) { FSHIntrinsic->setCalledFunction(FSHFunc); return; } auto *RotateBB = BasicBlock::Create(M->getContext(), "rotate", FSHFunc); IRBuilder<> Builder(RotateBB); Type *Ty = FSHFunc->getReturnType(); // Build the actual funnel shift rotate logic. // In the comments, "int" is used interchangeably with "vector of int // elements". FixedVectorType *VectorTy = dyn_cast(Ty); Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty; unsigned BitWidth = IntTy->getIntegerBitWidth(); ConstantInt *BitWidthConstant = Builder.getInt({BitWidth, BitWidth}); Value *BitWidthForInsts = VectorTy ? Builder.CreateVectorSplat(VectorTy->getNumElements(), BitWidthConstant) : BitWidthConstant; auto *RotateModVal = Builder.CreateURem(/*Rotate*/ FSHFunc->getArg(2), BitWidthForInsts); Value *FirstShift = nullptr, *SecShift = nullptr; if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) // Shift the less significant number right, the "rotate" number of bits // will be 0-filled on the left as a result of this regular shift. FirstShift = Builder.CreateLShr(FSHFunc->getArg(1), RotateModVal); else // Shift the more significant number left, the "rotate" number of bits // will be 0-filled on the right as a result of this regular shift. FirstShift = Builder.CreateShl(FSHFunc->getArg(0), RotateModVal); // We want the "rotate" number of the more significant int's LSBs (MSBs) to // occupy the leftmost (rightmost) "0 space" left by the previous operation. // Therefore, subtract the "rotate" number from the integer bitsize... auto *SubRotateVal = Builder.CreateSub(BitWidthForInsts, RotateModVal); if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) // ...and left-shift the more significant int by this number, zero-filling // the LSBs. SecShift = Builder.CreateShl(FSHFunc->getArg(0), SubRotateVal); else // ...and right-shift the less significant int by this number, zero-filling // the MSBs. SecShift = Builder.CreateLShr(FSHFunc->getArg(1), SubRotateVal); // A simple binary addition of the shifted ints yields the final result. auto *FunnelShiftRes = Builder.CreateOr(FirstShift, SecShift); Builder.CreateRet(FunnelShiftRes); FSHIntrinsic->setCalledFunction(FSHFunc); } void SPIRVRegularizeLLVMBase::buildUMulWithOverflowFunc(Function *UMulFunc) { if (!UMulFunc->empty()) return; BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", UMulFunc); IRBuilder<> Builder(EntryBB); // Build the actual unsigned multiplication logic with the overflow // indication. auto *FirstArg = UMulFunc->getArg(0); auto *SecondArg = UMulFunc->getArg(1); // Do unsigned multiplication Mul = A * B. // Then check if unsigned division Div = Mul / A is not equal to B. // If so, then overflow has happened. auto *Mul = Builder.CreateNUWMul(FirstArg, SecondArg); auto *Div = Builder.CreateUDiv(Mul, FirstArg); auto *Overflow = Builder.CreateICmpNE(FirstArg, Div); // umul.with.overflow intrinsic return a structure, where the first element // is the multiplication result, and the second is an overflow bit. auto *StructTy = UMulFunc->getReturnType(); auto *Agg = Builder.CreateInsertValue(UndefValue::get(StructTy), Mul, {0}); auto *Res = Builder.CreateInsertValue(Agg, Overflow, {1}); Builder.CreateRet(Res); } void SPIRVRegularizeLLVMBase::lowerUMulWithOverflow( IntrinsicInst *UMulIntrinsic) { // Get a separate function - otherwise, we'd have to rework the CFG of the // current one. Then simply replace the intrinsic uses with a call to the new // function. FunctionType *UMulFuncTy = UMulIntrinsic->getFunctionType(); Type *FSHLRetTy = UMulFuncTy->getReturnType(); const std::string FuncName = lowerLLVMIntrinsicName(UMulIntrinsic); Function *UMulFunc = getOrCreateFunction(M, FSHLRetTy, UMulFuncTy->params(), FuncName); buildUMulWithOverflowFunc(UMulFunc); UMulIntrinsic->setCalledFunction(UMulFunc); } void SPIRVRegularizeLLVMBase::expandVEDWithSYCLTypeSRetArg(Function *F) { auto Attrs = F->getAttributes(); StructType *SRetTy = cast(Attrs.getParamStructRetType(0)); Attrs = Attrs.removeParamAttribute(F->getContext(), 0, Attribute::StructRet); std::string Name = F->getName().str(); CallInst *OldCall = nullptr; mutateFunction( F, [=, &OldCall](CallInst *CI, std::vector &Args, Type *&RetTy) { Args.erase(Args.begin()); RetTy = SRetTy->getElementType(0); OldCall = CI; return Name; }, [=, &OldCall](CallInst *NewCI) { IRBuilder<> Builder(OldCall); Value *Target = Builder.CreateStructGEP(SRetTy, OldCall->getOperand(0), 0); return Builder.CreateStore(NewCI, Target); }, nullptr, &Attrs, true); } void SPIRVRegularizeLLVMBase::expandVIDWithSYCLTypeByValComp(Function *F) { auto Attrs = F->getAttributes(); auto *CompPtrTy = cast(Attrs.getParamByValType(1)); Attrs = Attrs.removeParamAttribute(F->getContext(), 1, Attribute::ByVal); std::string Name = F->getName().str(); mutateFunction( F, [=](CallInst *CI, std::vector &Args) { Type *HalfTy = CompPtrTy->getElementType(0); IRBuilder<> Builder(CI); auto *Target = Builder.CreateStructGEP(CompPtrTy, CI->getOperand(1), 0); Args[1] = Builder.CreateLoad(HalfTy, Target); return Name; }, nullptr, &Attrs, true); } void SPIRVRegularizeLLVMBase::expandSYCLTypeUsing(Module *M) { std::vector ToExpandVEDWithSYCLTypeSRetArg; std::vector ToExpandVIDWithSYCLTypeByValComp; for (auto &F : *M) { if (F.getName().startswith("_Z28__spirv_VectorExtractDynamic") && F.hasStructRetAttr()) { auto *SRetTy = F.getParamStructRetType(0); if (isSYCLHalfType(SRetTy) || isSYCLBfloat16Type(SRetTy)) ToExpandVEDWithSYCLTypeSRetArg.push_back(&F); else llvm_unreachable("The return type of the VectorExtractDynamic " "instruction cannot be a structure other than SYCL " "half."); } if (F.getName().startswith("_Z27__spirv_VectorInsertDynamic") && F.getArg(1)->getType()->isPointerTy()) { auto *ET = F.getParamByValType(1); if (isSYCLHalfType(ET) || isSYCLBfloat16Type(ET)) ToExpandVIDWithSYCLTypeByValComp.push_back(&F); else llvm_unreachable("The component argument type of an " "VectorInsertDynamic instruction can't be a " "structure other than SYCL half."); } } for (auto *F : ToExpandVEDWithSYCLTypeSRetArg) expandVEDWithSYCLTypeSRetArg(F); for (auto *F : ToExpandVIDWithSYCLTypeByValComp) expandVIDWithSYCLTypeByValComp(F); } Value *SPIRVRegularizeLLVMBase::extendBitInstBoolArg(Instruction *II) { IRBuilder<> Builder(II); auto *ArgTy = II->getOperand(0)->getType(); Type *NewArgType = nullptr; if (ArgTy->isIntegerTy()) { NewArgType = Builder.getInt32Ty(); } else if (ArgTy->isVectorTy() && cast(ArgTy)->getElementType()->isIntegerTy()) { unsigned NumElements = cast(ArgTy)->getNumElements(); NewArgType = VectorType::get(Builder.getInt32Ty(), NumElements, false); } else { llvm_unreachable("Unexpected type"); } auto *NewBase = Builder.CreateZExt(II->getOperand(0), NewArgType); auto *NewShift = Builder.CreateZExt(II->getOperand(1), NewArgType); switch (II->getOpcode()) { case Instruction::LShr: return Builder.CreateLShr(NewBase, NewShift); case Instruction::Shl: return Builder.CreateShl(NewBase, NewShift); default: return II; } } bool SPIRVRegularizeLLVMBase::runRegularizeLLVM(Module &Module) { M = &Module; Ctx = &M->getContext(); LLVM_DEBUG(dbgs() << "Enter SPIRVRegularizeLLVM:\n"); regularize(); LLVM_DEBUG(dbgs() << "After SPIRVRegularizeLLVM:\n" << *M); verifyRegularizationPass(*M, "SPIRVRegularizeLLVM"); return true; } /// Remove entities not representable by SPIR-V bool SPIRVRegularizeLLVMBase::regularize() { eraseUselessFunctions(M); addKernelEntryPoint(M); expandSYCLTypeUsing(M); for (auto I = M->begin(), E = M->end(); I != E;) { Function *F = &(*I++); if (F->isDeclaration() && F->use_empty()) { F->eraseFromParent(); continue; } std::vector ToErase; for (BasicBlock &BB : *F) { for (Instruction &II : BB) { if (auto Call = dyn_cast(&II)) { Call->setTailCall(false); Function *CF = Call->getCalledFunction(); if (CF && CF->isIntrinsic()) { removeFnAttr(Call, Attribute::NoUnwind); auto *II = cast(Call); if (II->getIntrinsicID() == Intrinsic::memset || II->getIntrinsicID() == Intrinsic::bswap) lowerIntrinsicToFunction(II); else if (II->getIntrinsicID() == Intrinsic::fshl || II->getIntrinsicID() == Intrinsic::fshr) lowerFunnelShift(II); else if (II->getIntrinsicID() == Intrinsic::umul_with_overflow) lowerUMulWithOverflow(II); } } // Translator treats i1 as boolean, but bit instructions take // a scalar/vector integers, so we have to extend such arguments if (II.isLogicalShift() && II.getOperand(0)->getType()->isIntOrIntVectorTy(1)) { auto *NewInst = extendBitInstBoolArg(&II); for (auto *U : II.users()) { if (cast(U)->getOpcode() == Instruction::ZExt) { U->dropAllReferences(); U->replaceAllUsesWith(NewInst); ToErase.push_back(cast(U)); } } ToErase.push_back(&II); } // Remove optimization info not supported by SPIRV if (auto BO = dyn_cast(&II)) { if (isa(BO) && BO->isExact()) BO->setIsExact(false); } // Remove metadata not supported by SPIRV static const char *MDs[] = { "fpmath", "tbaa", "range", }; for (auto &MDName : MDs) { if (II.getMetadata(MDName)) { II.setMetadata(MDName, nullptr); } } // Add an additional bitcast in case address space cast also changes // pointer element type. if (auto *ASCast = dyn_cast(&II)) { Type *DestTy = ASCast->getDestTy(); Type *SrcTy = ASCast->getSrcTy(); if (!II.getContext().supportsTypedPointers()) continue; if (DestTy->getScalarType()->getNonOpaquePointerElementType() != SrcTy->getScalarType()->getNonOpaquePointerElementType()) { Type *InterTy = PointerType::getWithSamePointeeType( cast(DestTy->getScalarType()), cast(SrcTy->getScalarType()) ->getPointerAddressSpace()); if (DestTy->isVectorTy()) InterTy = VectorType::get( InterTy, cast(DestTy)->getElementCount()); BitCastInst *NewBCast = new BitCastInst( ASCast->getPointerOperand(), InterTy, /*NameStr=*/"", ASCast); AddrSpaceCastInst *NewASCast = new AddrSpaceCastInst(NewBCast, DestTy, /*NameStr=*/"", ASCast); ToErase.push_back(ASCast); ASCast->dropAllReferences(); ASCast->replaceAllUsesWith(NewASCast); } } if (auto Cmpxchg = dyn_cast(&II)) { // Transform: // %1 = cmpxchg i32* %ptr, i32 %comparator, i32 %0 seq_cst acquire // To: // %cmpxchg.res = call spir_func // i32 @_Z29__spirv_AtomicCompareExchangePiiiiii( // i32* %ptr, i32 1, i32 16, i32 2, i32 %0, i32 %comparator) // %cmpxchg.success = icmp eq i32 %cmpxchg.res, %comparator // %1 = insertvalue { i32, i1 } undef, i32 %cmpxchg.res, 0 // %2 = insertvalue { i32, i1 } %1, i1 %cmpxchg.success, 1 // To get memory scope argument we might use Cmpxchg->getSyncScopeID() // but LLVM's cmpxchg instruction is not aware of OpenCL(or SPIR-V) // memory scope enumeration. And assuming the produced SPIR-V module // will be consumed in an OpenCL environment, we can use the same // memory scope as OpenCL atomic functions that do not have // memory_scope argument, i.e. memory_scope_device. See the OpenCL C // specification p6.13.11. Atomic Functions // cmpxchg LLVM instruction returns a pair {i32, i1}: the original // value and a flag indicating success (true) or failure (false). // OpAtomicCompareExchange SPIR-V instruction returns only the // original value. To keep the return type({i32, i1}) we construct // a composite. The first element of the composite holds result of // OpAtomicCompareExchange, i.e. the original value. The second // element holds result of comparison of the returned value and the // comparator, which matches with semantics of the flag returned by // cmpxchg. Value *Ptr = Cmpxchg->getPointerOperand(); Value *MemoryScope = getInt32(M, spv::ScopeDevice); auto SuccessOrder = static_cast( llvm::toCABI(Cmpxchg->getSuccessOrdering())); auto FailureOrder = static_cast( llvm::toCABI(Cmpxchg->getFailureOrdering())); Value *EqualSem = getInt32(M, OCLMemOrderMap::map(SuccessOrder)); Value *UnequalSem = getInt32(M, OCLMemOrderMap::map(FailureOrder)); Value *Val = Cmpxchg->getNewValOperand(); Value *Comparator = Cmpxchg->getCompareOperand(); Type *MemType = Cmpxchg->getCompareOperand()->getType(); llvm::Value *Args[] = {Ptr, MemoryScope, EqualSem, UnequalSem, Val, Comparator}; auto *Res = addCallInstSPIRV(M, "__spirv_AtomicCompareExchange", MemType, Args, nullptr, {MemType}, &II, "cmpxchg.res"); IRBuilder<> Builder(Cmpxchg); auto *Cmp = Builder.CreateICmpEQ(Res, Comparator, "cmpxchg.success"); auto *V1 = Builder.CreateInsertValue( UndefValue::get(Cmpxchg->getType()), Res, 0); auto *V2 = Builder.CreateInsertValue(V1, Cmp, 1, Cmpxchg->getName()); Cmpxchg->replaceAllUsesWith(V2); ToErase.push_back(Cmpxchg); } } } for (Instruction *V : ToErase) { assert(V->user_empty()); V->eraseFromParent(); } } if (SPIRVDbgSaveRegularizedModule) saveLLVMModule(M, RegularizedModuleTmpFile); return true; } void SPIRVRegularizeLLVMBase::addKernelEntryPoint(Module *M) { std::vector Work; // Get a list of all functions that have SPIR kernel calling conv for (auto &F : *M) { if (F.getCallingConv() == CallingConv::SPIR_KERNEL) Work.push_back(&F); } for (auto &F : Work) { // for declarations just make them into SPIR functions. F->setCallingConv(CallingConv::SPIR_FUNC); if (F->isDeclaration()) continue; // Otherwise add a wrapper around the function to act as an entry point. FunctionType *FType = F->getFunctionType(); std::string WrapName = kSPIRVName::EntrypointPrefix + static_cast(F->getName()); Function *WrapFn = getOrCreateFunction(M, F->getReturnType(), FType->params(), WrapName); auto *CallBB = BasicBlock::Create(M->getContext(), "", WrapFn); IRBuilder<> Builder(CallBB); Function::arg_iterator DestI = WrapFn->arg_begin(); for (const Argument &I : F->args()) { DestI->setName(I.getName()); DestI++; } SmallVector Args; for (Argument &I : WrapFn->args()) { Args.emplace_back(&I); } auto *CI = CallInst::Create(F, ArrayRef(Args), "", CallBB); CI->setCallingConv(F->getCallingConv()); CI->setAttributes(F->getAttributes()); // copy over all the metadata (should it be removed from F?) SmallVector> MDs; F->getAllMetadata(MDs); WrapFn->setAttributes(F->getAttributes()); for (auto MD = MDs.begin(), End = MDs.end(); MD != End; ++MD) { WrapFn->addMetadata(MD->first, *MD->second); } WrapFn->setCallingConv(CallingConv::SPIR_KERNEL); WrapFn->setLinkage(llvm::GlobalValue::InternalLinkage); Builder.CreateRet(F->getReturnType()->isVoidTy() ? nullptr : CI); // Have to find the spir-v metadata for execution mode and transfer it to // the wrapper. if (auto NMD = SPIRVMDWalker(*M).getNamedMD(kSPIRVMD::ExecutionMode)) { while (!NMD.atEnd()) { Function *MDF = nullptr; auto N = NMD.nextOp(); /* execution mode MDNode */ N.get(MDF); if (MDF == F) N.M->replaceOperandWith(0, ValueAsMetadata::get(WrapFn)); } } } } } // namespace SPIRV INITIALIZE_PASS(SPIRVRegularizeLLVMLegacy, "spvregular", "Regularize LLVM for SPIR-V", false, false) ModulePass *llvm::createSPIRVRegularizeLLVMLegacy() { return new SPIRVRegularizeLLVMLegacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVRegularizeLLVM.h000066400000000000000000000134341506646577400227720ustar00rootroot00000000000000//=- SPIRVRegularizeLLVM.h - LLVM Module regularization pass -*- C++ -*-=// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVREGULARIZELLVM_H #define SPIRV_SPIRVREGULARIZELLVM_H #include "SPIRVInternal.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" namespace SPIRV { class SPIRVRegularizeLLVMBase { public: SPIRVRegularizeLLVMBase() : M(nullptr), Ctx(nullptr) {} bool runRegularizeLLVM(llvm::Module &M); // Lower functions bool regularize(); // SPIR-V disallows functions being entrypoints and called // LLVM doesn't. This adds a wrapper around the entry point // that later SPIR-V writer renames. void addKernelEntryPoint(llvm::Module *M); /// Some LLVM intrinsics that have no SPIR-V counterpart may be wrapped in /// @spirv.llvm_intrinsic_* function. During reverse translation from SPIR-V /// to LLVM IR we can detect this @spirv.llvm_intrinsic_* function and /// replace it with @llvm.intrinsic.* back. void lowerIntrinsicToFunction(llvm::IntrinsicInst *Intrinsic); /// No SPIR-V counterpart for @llvm.fshl.*(@llvm.fshr.*) intrinsic. It will be /// lowered to a newly generated @spirv.llvm_fshl_*(@spirv.llvm_fshr_*) /// function. /// /// Conceptually, FSHL (FSHR): /// 1. concatenates the ints, the first one being the more significant; /// 2. performs a left (right) shift-rotate on the resulting doubled-sized /// int; /// 3. returns the most (least) significant bits of the shift-rotate result, /// the number of bits being equal to the size of the original integers. /// If FSHL (FSHR) operates on a vector type instead, the same operations are /// performed for each set of corresponding vector elements. /// /// The actual implementation algorithm will be slightly different for /// simplification purposes. void lowerFunnelShift(llvm::IntrinsicInst *FSHIntrinsic); void lowerUMulWithOverflow(llvm::IntrinsicInst *UMulIntrinsic); void buildUMulWithOverflowFunc(llvm::Function *UMulFunc); // For some cases Clang emits VectorExtractDynamic as: // void @_Z28__spirv_VectorExtractDynamic(* sret(), jointMatrix, idx); // Instead of: // @_Z28__spirv_VectorExtractDynamic(JointMatrix, Idx); // And VectorInsertDynamic as: // @_Z27__spirv_VectorInsertDynamic(jointMatrix, * byval(), idx); // Instead of: // @_Z27__spirv_VectorInsertDynamic(jointMatrix, , idx) // Need to add additional GEP, store and load instructions and mutate called // function to avoid translation failures void expandSYCLTypeUsing(llvm::Module *M); void expandVEDWithSYCLTypeSRetArg(llvm::Function *F); void expandVIDWithSYCLTypeByValComp(llvm::Function *F); // According to the specification, the operands of a shift instruction must be // a scalar/vector of integer. When LLVM-IR contains a shift instruction with // i1 operands, they are treated as a bool. We need to extend them to i32 to // comply with the specification. For example: "%shift = lshr i1 0, 1"; // The bit instruction should be changed to the extended version // "%shift = lshr i32 0, 1" so the args are treated as int operands. Value *extendBitInstBoolArg(llvm::Instruction *OldInst); static std::string lowerLLVMIntrinsicName(llvm::IntrinsicInst *II); static char ID; private: llvm::Module *M; llvm::LLVMContext *Ctx; }; class SPIRVRegularizeLLVMPass : public llvm::PassInfoMixin, public SPIRVRegularizeLLVMBase { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runRegularizeLLVM(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } }; class SPIRVRegularizeLLVMLegacy : public llvm::ModulePass, public SPIRVRegularizeLLVMBase { public: SPIRVRegularizeLLVMLegacy() : ModulePass(ID) { initializeSPIRVRegularizeLLVMLegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(llvm::Module &M) override; static char ID; }; } // namespace SPIRV #endif // SPIRV_SPIRVREGULARIZELLVM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToLLVMDbgTran.cpp000066400000000000000000002324331506646577400230420ustar00rootroot00000000000000//===- SPIRVToLLVMDbgTran.cpp - Converts debug info to LLVM -----*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2018 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements translation of debug info from SPIR-V to LLVM metadata // //===----------------------------------------------------------------------===// #include "SPIRVToLLVMDbgTran.h" #include "SPIRV.debug.h" #include "SPIRVEntry.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVInternal.h" #include "SPIRVReader.h" #include "SPIRVType.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" using namespace std; using namespace SPIRVDebug::Operand; namespace SPIRV { static uint64_t getDerivedSizeInBits(const DIType *Ty) { if (auto Size = Ty->getSizeInBits()) return Size; if (auto *DT = llvm::dyn_cast(Ty)) if (auto *BT = llvm::dyn_cast(DT->getRawBaseType())) return getDerivedSizeInBits(BT); return 0; } SPIRVToLLVMDbgTran::SPIRVToLLVMDbgTran(SPIRVModule *TBM, Module *TM, SPIRVToLLVM *Reader) : BM(TBM), M(TM), SPIRVReader(Reader) { Enable = BM->hasDebugInfo(); } void SPIRVToLLVMDbgTran::addDbgInfoVersion() { if (!Enable) return; M->addModuleFlag(Module::Warning, "Debug Info Version", DEBUG_METADATA_VERSION); } DIFile * SPIRVToLLVMDbgTran::getDIFile(const std::string &FileName, Optional> CS, Optional Source) { return getOrInsert(FileMap, FileName, [=]() { SplitFileName Split(FileName); // Use the first builder from the map to crete DIFile since it's // relations with other debug metadata is not going through DICompileUnit if (!Split.BaseName.empty()) { if (Source.hasValue()) { const StringRef SourceRef = StringRef(Source.getValue()); return BuilderMap.begin()->second->createFile( Split.BaseName, Split.Path, CS, SourceRef); } return BuilderMap.begin()->second->createFile(Split.BaseName, Split.Path, CS); } return static_cast(nullptr); }); } SPIRVExtInst *SPIRVToLLVMDbgTran::getDbgInst(const SPIRVId Id) { SPIRVEntry *E = BM->getEntry(Id); if (isa(E)) { SPIRVExtInst *EI = static_cast(E); if (EI->getExtSetKind() == SPIRV::SPIRVEIS_Debug || EI->getExtSetKind() == SPIRV::SPIRVEIS_OpenCL_DebugInfo_100 || EI->getExtSetKind() == SPIRV::SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || EI->getExtSetKind() == SPIRV::SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return EI; } return nullptr; } // Check if module is generated using NonSemantic.Shader.DebugInfo.XXX // extended instruction set inline bool isNonSemanticDebugInfo(const SPIRVExtInstSetKind Kind) { return (Kind == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || Kind == SPIRVEIS_NonSemantic_Shader_DebugInfo_200); } // Get integer parameter of debug instruction considering whether it's // Literal or of OpConstant instruction depending on DebugInfo // extended instruction set kind SPIRVWord SPIRVToLLVMDbgTran::getConstantValueOrLiteral(const std::vector &Ops, const SPIRVWord Idx, const SPIRVExtInstSetKind Kind) { if (!isNonSemanticDebugInfo(Kind)) return Ops[Idx]; SPIRVValue *SPVConst = BM->get(Ops[Idx]); assert(isConstantOpCode(SPVConst->getOpCode()) && "NonSemantic Debug instruction's parameters must be OpConstant"); ConstantInt *Const = cast(SPIRVReader->transValue(SPVConst, nullptr, nullptr)); return Const->getZExtValue(); } const std::string &SPIRVToLLVMDbgTran::getString(const SPIRVId Id) { SPIRVString *String = BM->get(Id); assert(String && "Invalid string"); return String->getStr(); } Optional SPIRVToLLVMDbgTran::getStringSourceContinued(const SPIRVId Id, SPIRVExtInst *DebugInst) { if (!isValidId(Id) || getDbgInst(Id)) return {}; std::string Str = BM->get(Id)->getStr(); using namespace SPIRVDebug::Operand::SourceContinued; for (auto *I : DebugInst->getContinuedInstructions()) { const std::string TmpStr = BM->get(I->getArguments()[TextIdx])->getStr(); Str.append(TmpStr); } return Str; } void SPIRVToLLVMDbgTran::transDbgInfo(const SPIRVValue *SV, Value *V) { // Constant samplers and composites do not have a corresponding // SPIRVInstruction, but they may be mapped to an LLVM Instruction. if (SV->getOpCode() == OpConstantSampler || SV->getOpCode() == OpConstantComposite) return; if (Instruction *I = dyn_cast(V)) { const SPIRVInstruction *SI = static_cast(SV); I->setDebugLoc(transDebugScope(SI)); } } DIScope *SPIRVToLLVMDbgTran::getScope(const SPIRVEntry *ScopeInst) { if (ScopeInst->getOpCode() == OpString) return getDIFile(static_cast(ScopeInst)->getStr()); return transDebugInst(static_cast(ScopeInst)); } void SPIRVToLLVMDbgTran::appendToSourceLangLiteral(DICompileUnit *CompileUnit, SPIRVWord SourceLang) { if (!M->getModuleFlag("Source Lang Literal")) { M->addModuleFlag(llvm::Module::Warning, "Source Lang Literal", MDTuple::get(M->getContext(), {})); } auto *SourceLangLiteral = dyn_cast(M->getModuleFlag("Source Lang Literal")); // Copy old content SmallVector Nodes; for (auto &Node : SourceLangLiteral->operands()) { Nodes.push_back(Node); } // Add new entry Nodes.push_back(MDTuple::get( M->getContext(), SmallVector{ CompileUnit, ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(M->getContext()), SourceLang)), })); // Update M->setModuleFlag(llvm::Module::Warning, "Source Lang Literal", MDTuple::get(M->getContext(), Nodes)); } DICompileUnit * SPIRVToLLVMDbgTran::transCompilationUnit(const SPIRVExtInst *DebugInst, const std::string CompilerVersion, const std::string Flags) { // Do nothing in case we have already translated the CU (e.g. during // DebugEntryPoint translation) if (BuilderMap[DebugInst->getId()]) return cast(DebugInstCache[DebugInst]); const SPIRVWordVec &Ops = DebugInst->getArguments(); using namespace SPIRVDebug::Operand::CompilationUnit; assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); // We must preserve only one Dwarf version module level metadata // UpgradeDebugInfo from llvm/lib/IR/AutoUpgrade.cpp has already done all // work for us during linking stage leaving a single Dwarf version in the // module if (!M->getModuleFlag("Dwarf Version")) { SPIRVWord DWARFVersion = getConstantValueOrLiteral( Ops, DWARFVersionIdx, DebugInst->getExtSetKind()); M->addModuleFlag(llvm::Module::Max, "Dwarf Version", DWARFVersion); } SPIRVWord SourceLang = getConstantValueOrLiteral(Ops, LanguageIdx, DebugInst->getExtSetKind()); SPIRVWord OriginalSourceLang = SourceLang; bool InvalidSourceLang = false; if (DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { SourceLang = convertSPIRVSourceLangToDWARFNonSemanticDbgInfo(SourceLang); } else if (isSPIRVSourceLangValid(SourceLang)) { SourceLang = convertSPIRVSourceLangToDWARF(SourceLang); } else { // Some SPIR-V producers generate invalid source language value. In such // case the original value should be preserved in "Source Lang Literal" // module flag for later use by LLVM IR consumers. SourceLang = dwarf::DW_LANG_OpenCL; InvalidSourceLang = true; } BuilderMap[DebugInst->getId()] = std::make_unique(*M); assert(BuilderMap.size() != 0 && "No debug compile units"); if (isNonSemanticDebugInfo(DebugInst->getExtSetKind())) { // Only initialize once if (BuilderMap.size() == 1) setBuildIdentifierAndStoragePath(); // This assertion is a guard/reminder to update the Producer argument to // createCompileUnit() if a new DebugInfo type is ever created assert(DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200); auto *CompileUnit = BuilderMap[DebugInst->getId()]->createCompileUnit( SourceLang, getFile(Ops[SourceIdx]), DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 ? CompilerVersion : getString(Ops[ProducerIdx]), false, Flags, 0, StoragePath, DICompileUnit::DebugEmissionKind::FullDebug, BuildIdentifier); if (InvalidSourceLang) { appendToSourceLangLiteral(CompileUnit, OriginalSourceLang); } return CompileUnit; } // TODO: Remove this workaround once we switch to NonSemantic.Shader.* debug // info by default auto Producer = findModuleProducer(); auto *CompileUnit = BuilderMap[DebugInst->getId()]->createCompileUnit( SourceLang, getFile(Ops[SourceIdx]), Producer, false, Flags, 0); if (InvalidSourceLang) { appendToSourceLangLiteral(CompileUnit, OriginalSourceLang); } return CompileUnit; } DIBasicType *SPIRVToLLVMDbgTran::transTypeBasic(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeBasic; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert((Ops.size() == OperandCountOCL || Ops.size() == OperandCountNonSemantic) && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); auto Tag = static_cast( getConstantValueOrLiteral(Ops, EncodingIdx, DebugInst->getExtSetKind())); unsigned Encoding = SPIRV::DbgEncodingMap::rmap(Tag); if (Encoding == 0) return getDIBuilder(DebugInst).createUnspecifiedType(Name); uint64_t Size = BM->get(Ops[SizeIdx])->getZExtIntValue(); return getDIBuilder(DebugInst).createBasicType(Name, Size, Encoding); } DIDerivedType * SPIRVToLLVMDbgTran::transTypeQualifier(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeQualifier; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() == OperandCount && "Invalid number of operands"); DIType *BaseTy = transDebugInst(BM->get(Ops[BaseTypeIdx])); SPIRVWord Tag = SPIRV::DbgTypeQulifierMap::rmap( static_cast(getConstantValueOrLiteral( Ops, QualifierIdx, DebugInst->getExtSetKind()))); return getDIBuilder(DebugInst).createQualifiedType(Tag, BaseTy); } DIType *SPIRVToLLVMDbgTran::transTypePointer(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypePointer; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() == OperandCount && "Invalid number of operands"); DIType *PointeeTy = nullptr; if (BM->getEntry(Ops[BaseTypeIdx])->getOpCode() != OpTypeVoid) PointeeTy = transDebugInst(BM->get(Ops[BaseTypeIdx])); llvm::Optional AS; SPIRVWord SC = getConstantValueOrLiteral(Ops, StorageClassIdx, DebugInst->getExtSetKind()); if (SC != ~0U) // all ones denote no address space AS = SPIRSPIRVAddrSpaceMap::rmap(static_cast(SC)); DIType *Ty; SPIRVWord Flags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); if (Flags & SPIRVDebug::FlagIsLValueReference) Ty = getDIBuilder(DebugInst).createReferenceType( dwarf::DW_TAG_reference_type, PointeeTy, 0, 0, AS); else if (Flags & SPIRVDebug::FlagIsRValueReference) Ty = getDIBuilder(DebugInst).createReferenceType( dwarf::DW_TAG_rvalue_reference_type, PointeeTy, 0, 0, AS); else Ty = getDIBuilder(DebugInst).createPointerType( PointeeTy, BM->getAddressingModel() * 32, 0, AS); if (Flags & SPIRVDebug::FlagIsObjectPointer) Ty = getDIBuilder(DebugInst).createObjectPointerType(Ty); else if (Flags & SPIRVDebug::FlagIsArtificial) Ty = getDIBuilder(DebugInst).createArtificialType(Ty); return Ty; } DICompositeType * SPIRVToLLVMDbgTran::transTypeArray(const SPIRVExtInst *DebugInst) { if (DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transTypeArrayNonSemantic(DebugInst); // TODO: figure out better naming for transTypeArrayOpenCL since // it also handles SPIRVEIS_NonSemantic_Shader_DebugInfo_100. // Also to consider separating OpenCL from NonSemantic, as OpenCL has several // workarounds return transTypeArrayOpenCL(DebugInst); } DICompositeType * SPIRVToLLVMDbgTran::transTypeArrayOpenCL(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeArray; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIType *BaseTy = transNonNullDebugType(BM->get(Ops[BaseTypeIdx])); size_t TotalCount = 1; SmallVector Subscripts; // Ops looks like: { BaseType, count1|upperBound1, count2|upperBound2, ..., // countN|upperBoundN, lowerBound1, lowerBound2, ..., lowerBoundN } for (size_t I = ComponentCountIdx, E = Ops.size() / 2 + 1; I < E; ++I) { if (auto *LocalVar = getDbgInst(Ops[I])) { auto *UpperBound = transDebugInst(LocalVar); SPIRVConstant *C = BM->get(Ops[Ops.size() / 2 + I]); int64_t ConstantAsInt = static_cast(C->getZExtIntValue()); auto *LowerBound = ConstantAsMetadata::get( ConstantInt::get(M->getContext(), APInt(64, ConstantAsInt))); Subscripts.push_back(getDIBuilder(DebugInst).getOrCreateSubrange( nullptr, LowerBound, UpperBound, nullptr)); continue; } const SPIRVExtInst *ExprUB, *ExprLB; if ((ExprUB = getDbgInst(Ops[I])) && (ExprLB = getDbgInst(Ops[Ops.size() / 2 + I]))) { auto *UpperBound = transDebugInst(ExprUB); auto *LowerBound = transDebugInst(ExprLB); Subscripts.push_back(getDIBuilder(DebugInst).getOrCreateSubrange( nullptr, LowerBound, UpperBound, nullptr)); continue; } if (!getDbgInst(Ops[I])) { SPIRVConstant *C = BM->get(Ops[I]); int64_t Count = static_cast(C->getZExtIntValue()); // If the SPIR-V file was generated by an older version of the translator, // Ops may not contain the LowerBound if (Ops.size() / 2 + I < Ops.size()) { C = BM->get(Ops[Ops.size() / 2 + I]); int64_t LowerBound = static_cast(C->getZExtIntValue()); Subscripts.push_back( getDIBuilder(DebugInst).getOrCreateSubrange(LowerBound, Count)); } else { auto *CountAsMD = ConstantAsMetadata::get( ConstantInt::get(M->getContext(), APInt(64, Count))); Subscripts.push_back(getDIBuilder(DebugInst).getOrCreateSubrange( CountAsMD, nullptr, nullptr, nullptr)); } // Count = -1 means that the array is empty TotalCount *= Count > 0 ? static_cast(Count) : 0; continue; } } DINodeArray SubscriptArray = getDIBuilder(DebugInst).getOrCreateArray(Subscripts); size_t Size = getDerivedSizeInBits(BaseTy) * TotalCount; return getDIBuilder(DebugInst).createArrayType(Size, 0 /*align*/, BaseTy, SubscriptArray); } DICompositeType * SPIRVToLLVMDbgTran::transTypeArrayNonSemantic(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeArray; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIType *BaseTy = transNonNullDebugType(BM->get(Ops[BaseTypeIdx])); size_t TotalCount = 1; SmallVector Subscripts; if (DebugInst->getExtOp() == SPIRVDebug::TypeArray) { for (size_t I = SubrangesIdx; I < Ops.size(); ++I) { auto *SR = transDebugInst(BM->get(Ops[I])); // LLVM 15 - exclusive workaround, get from a pointer union seem to be // bugged if (isa(SR->getCount())) if (auto *Count = SR->getCount().get()) TotalCount *= Count->getSExtValue() > 0 ? Count->getSExtValue() : 0; Subscripts.push_back(SR); } } DINodeArray SubscriptArray = getDIBuilder(DebugInst).getOrCreateArray(Subscripts); size_t Size = getDerivedSizeInBits(BaseTy) * TotalCount; return getDIBuilder(DebugInst).createArrayType(Size, 0 /*align*/, BaseTy, SubscriptArray); } DICompositeType * SPIRVToLLVMDbgTran::transTypeArrayDynamic(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeArrayDynamic; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIType *BaseTy = transNonNullDebugType(BM->get(Ops[BaseTypeIdx])); size_t TotalCount = 1; SmallVector Subscripts; for (size_t I = SubrangesIdx; I < Ops.size(); ++I) { auto *SR = transDebugInst(BM->get(Ops[I])); // LLVM 15 - exclusive workaround, get from a pointer union seem to be // bugged if (isa(SR->getCount())) if (auto *Count = SR->getCount().get()) TotalCount *= Count->getSExtValue() > 0 ? Count->getSExtValue() : 0; Subscripts.push_back(SR); } DINodeArray SubscriptArray = getDIBuilder(DebugInst).getOrCreateArray(Subscripts); size_t Size = getDerivedSizeInBits(BaseTy) * TotalCount; auto TransOperand = [&](SPIRVWord Idx) -> PointerUnion { if (!getDbgInst(Ops[Idx])) { if (const auto *GV = getDbgInst(Ops[Idx])) return transDebugInst(GV); if (const auto *LV = getDbgInst(Ops[Idx])) return transDebugInst(LV); if (const auto *DIExpr = getDbgInst(Ops[Idx])) return transDebugInst(DIExpr); } return nullptr; }; PointerUnion DataLocation = TransOperand(DataLocationIdx); PointerUnion Associated = TransOperand(AssociatedIdx); PointerUnion Allocated = TransOperand(AllocatedIdx); PointerUnion Rank = TransOperand(RankIdx); return getDIBuilder(DebugInst).createArrayType(Size, 0 /*align*/, BaseTy, SubscriptArray, DataLocation, Associated, Allocated, Rank); } DICompositeType * SPIRVToLLVMDbgTran::transTypeVector(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeVector; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIType *BaseTy = transNonNullDebugType(BM->get(Ops[BaseTypeIdx])); SPIRVWord Count = getConstantValueOrLiteral(Ops, ComponentCountIdx, DebugInst->getExtSetKind()); // Round up to a power of two. // OpenCL/SYCL 3-element vectors // occupy the same amount of memory as 4-element vectors // Clang rounds up the memory size of vectors to a power of 2. // Vulkan allows vec3 to have a memory size of 12, but in RenderDoc memory // size is not derived from debug info. const uint64_t Size = getDerivedSizeInBits(BaseTy) * bitCeil(Count); SmallVector Subscripts; Subscripts.push_back(getDIBuilder(DebugInst).getOrCreateSubrange(0, Count)); DINodeArray SubscriptArray = getDIBuilder(DebugInst).getOrCreateArray(Subscripts); return getDIBuilder(DebugInst).createVectorType(Size, 0 /*align*/, BaseTy, SubscriptArray); } DICompositeType * SPIRVToLLVMDbgTran::transTypeComposite(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeComposite; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIScope *ParentScope = getScope(BM->getEntry(Ops[ParentIdx])); uint64_t Size = 0; SPIRVEntry *SizeEntry = BM->getEntry(Ops[SizeIdx]); if (!(SizeEntry->isExtInst(SPIRVEIS_Debug, SPIRVDebug::DebugInfoNone) || SizeEntry->isExtInst(SPIRVEIS_OpenCL_DebugInfo_100, SPIRVDebug::DebugInfoNone) || SizeEntry->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugInfoNone))) { Size = BM->get(Ops[SizeIdx])->getZExtIntValue(); } uint64_t Align = 0; DIType *DerivedFrom = nullptr; StringRef Identifier; SPIRVEntry *UniqId = BM->get(Ops[LinkageNameIdx]); if (UniqId->getOpCode() == OpString) Identifier = static_cast(UniqId)->getStr(); DINode::DIFlags Flags = DINode::FlagZero; SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); if (SPIRVFlags & SPIRVDebug::FlagIsFwdDecl) Flags |= DINode::FlagFwdDecl; if (SPIRVFlags & SPIRVDebug::FlagTypePassByValue) Flags |= DINode::FlagTypePassByValue; if (SPIRVFlags & SPIRVDebug::FlagTypePassByReference) Flags |= DINode::FlagTypePassByReference; DICompositeType *CT = nullptr; switch (getConstantValueOrLiteral(Ops, TagIdx, DebugInst->getExtSetKind())) { case SPIRVDebug::Class: // TODO: should be replaced with createClassType, when bug with creating // ClassType with llvm::dwarf::DW_TAG_struct_type tag will be fixed CT = getDIBuilder(DebugInst).createReplaceableCompositeType( llvm::dwarf::DW_TAG_class_type, Name, ParentScope, File, LineNo, 0, Size, Align, Flags, Identifier); CT = llvm::MDNode::replaceWithDistinct(llvm::TempDICompositeType(CT)); break; case SPIRVDebug::Structure: CT = getDIBuilder(DebugInst).createStructType( ParentScope, Name, File, LineNo, Size, Align, Flags, DerivedFrom, DINodeArray() /*elements*/, 0 /*RunTimeLang*/, nullptr /*VTableHolder*/, Identifier); break; case SPIRVDebug::Union: CT = getDIBuilder(DebugInst).createUnionType( ParentScope, Name, File, LineNo, Size, Align, Flags, DINodeArray(), 0 /*RuntimrLang*/, Identifier); break; default: llvm_unreachable("Unexpected composite type"); break; } DebugInstCache[DebugInst] = CT; SmallVector EltTys; for (size_t I = FirstMemberIdx; I < Ops.size(); ++I) { auto *MemberInst = BM->get(Ops[I]); if (MemberInst->getExtOp() == SPIRVDebug::TypeMember) { auto *SPVMemberInst = BM->get(Ops[I]); DINode *MemberMD = transTypeMember(SPVMemberInst, DebugInst, cast(CT)); EltTys.push_back(MemberMD); DebugInstCache[SPVMemberInst] = MemberMD; } else if (MemberInst->getExtOp() == SPIRVDebug::TypeInheritance) { auto *SPVMemberInst = BM->get(Ops[I]); DINode *MemberMD = transTypeInheritance(SPVMemberInst, cast(CT)); EltTys.push_back(MemberMD); DebugInstCache[SPVMemberInst] = MemberMD; } else { EltTys.emplace_back(transDebugInst(BM->get(Ops[I]))); } } DINodeArray Elements = getDIBuilder(DebugInst).getOrCreateArray(EltTys); getDIBuilder(DebugInst).replaceArrays(CT, Elements); assert(CT && "Composite type translation failed."); return CT; } DISubrange * SPIRVToLLVMDbgTran::transTypeSubrange(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeSubrange; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert((Ops.size() == MinOperandCount || Ops.size() == MaxOperandCount) && "Invalid number of operands"); std::vector TranslatedOps(MaxOperandCount, nullptr); auto TransOperand = [&Ops, &TranslatedOps, this](int Idx) -> void { if (!getDbgInst(Ops[Idx])) { if (auto *GlobalVar = getDbgInst(Ops[Idx])) { TranslatedOps[Idx] = cast(transDebugInst(GlobalVar)); } else if (auto *LocalVar = getDbgInst(Ops[Idx])) { TranslatedOps[Idx] = cast(transDebugInst(LocalVar)); } else if (auto *Expr = getDbgInst(Ops[Idx])) { TranslatedOps[Idx] = cast(transDebugInst(Expr)); } else if (auto *Const = BM->get(Ops[Idx])) { int64_t ConstantAsInt = static_cast(Const->getZExtIntValue()); TranslatedOps[Idx] = cast(ConstantAsMetadata::get( ConstantInt::get(M->getContext(), APInt(64, ConstantAsInt)))); } } }; for (size_t Idx = 0; Idx < Ops.size(); ++Idx) TransOperand(Idx); return getDIBuilder(DebugInst).getOrCreateSubrange( TranslatedOps[CountIdx], TranslatedOps[LowerBoundIdx], TranslatedOps[UpperBoundIdx], TranslatedOps[StrideIdx]); } DIStringType * SPIRVToLLVMDbgTran::transTypeString(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeString; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); unsigned Encoding = 0; if (!getDbgInst((Ops[BaseTypeIdx]))) { DIBasicType *BaseTy = transTypeBasic(BM->get(Ops[BaseTypeIdx])); Encoding = BaseTy->getEncoding(); } DIExpression *StrLocationExp = nullptr; if (!getDbgInst(Ops[DataLocationIdx])) { if (const auto *DIExpr = getDbgInst(Ops[DataLocationIdx])) StrLocationExp = transDebugInst(DIExpr); } uint64_t SizeInBits = BM->get(Ops[SizeIdx])->getZExtIntValue(); DIExpression *StringLengthExp = nullptr; DIVariable *StringLengthVar = nullptr; if (!getDbgInst(Ops[LengthAddrIdx])) { if (const auto *GV = getDbgInst(Ops[LengthAddrIdx])) StringLengthVar = transDebugInst(GV); if (const auto *LV = getDbgInst(Ops[LengthAddrIdx])) StringLengthVar = transDebugInst(LV); if (const auto *DIExpr = getDbgInst(Ops[LengthAddrIdx])) StringLengthExp = transDebugInst(DIExpr); } return DIStringType::get(M->getContext(), dwarf::DW_TAG_string_type, Name, cast_or_null(StringLengthVar), cast_or_null(StringLengthExp), cast_or_null(StrLocationExp), SizeInBits, 0 /*AlignInBits*/, Encoding); } DINode *SPIRVToLLVMDbgTran::transTypeMember(const SPIRVExtInst *DebugInst, const SPIRVExtInst *ParentInst, DIScope *Scope) { if (isNonSemanticDebugInfo(DebugInst->getExtSetKind())) // In NonSemantic spec TypeMember doesn't have Scope parameter return transTypeMemberNonSemantic(DebugInst, ParentInst, Scope); return transTypeMemberOpenCL(DebugInst); } DINode * SPIRVToLLVMDbgTran::transTypeMemberOpenCL(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeMember::OpenCL; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); StringRef Name = getString(Ops[NameIdx]); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); DIType *BaseType = transNonNullDebugType(BM->get(Ops[TypeIdx])); uint64_t OffsetInBits = BM->get(Ops[OffsetIdx])->getZExtIntValue(); SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); DINode::DIFlags Flags = DINode::FlagZero; if ((SPIRVDebug::FlagAccess & SPIRVFlags) == SPIRVDebug::FlagIsPublic) { Flags |= DINode::FlagPublic; } else if (SPIRVFlags & SPIRVDebug::FlagIsProtected) { Flags |= DINode::FlagProtected; } else if (SPIRVFlags & SPIRVDebug::FlagIsPrivate) { Flags |= DINode::FlagPrivate; } if (SPIRVFlags & SPIRVDebug::FlagIsStaticMember) Flags |= DINode::FlagStaticMember; if (Flags & DINode::FlagStaticMember && Ops.size() > MinOperandCount) { SPIRVValue *ConstVal = BM->get(Ops[ValueIdx]); assert(isConstantOpCode(ConstVal->getOpCode()) && "Static member must be a constant"); llvm::Value *Val = SPIRVReader->transValue(ConstVal, nullptr, nullptr); return getDIBuilder(DebugInst).createStaticMemberType( Scope, Name, File, LineNo, BaseType, Flags, cast(Val)); } uint64_t Size = BM->get(Ops[SizeIdx])->getZExtIntValue(); uint64_t Alignment = 0; return getDIBuilder(DebugInst).createMemberType(Scope, Name, File, LineNo, Size, Alignment, OffsetInBits, Flags, BaseType); } DINode * SPIRVToLLVMDbgTran::transTypeMemberNonSemantic(const SPIRVExtInst *DebugInst, const SPIRVExtInst *ParentInst, DIScope *Scope) { if (!Scope) // Will be translated later when processing TypeMember's parent return nullptr; using namespace SPIRVDebug::Operand::TypeMember::NonSemantic; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIFile *File = getFile(Ops[SourceIdx]); const SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); const StringRef Name = getString(Ops[NameIdx]); DIType *BaseType = transNonNullDebugType(BM->get(Ops[TypeIdx])); const uint64_t OffsetInBits = BM->get(Ops[OffsetIdx])->getZExtIntValue(); const SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); DINode::DIFlags Flags = DINode::FlagZero; if ((SPIRVDebug::FlagAccess & SPIRVFlags) == SPIRVDebug::FlagIsPublic) { Flags |= DINode::FlagPublic; } else if (SPIRVFlags & SPIRVDebug::FlagIsProtected) { Flags |= DINode::FlagProtected; } else if (SPIRVFlags & SPIRVDebug::FlagIsPrivate) { Flags |= DINode::FlagPrivate; } if (SPIRVFlags & SPIRVDebug::FlagIsStaticMember) Flags |= DINode::FlagStaticMember; if (SPIRVFlags & SPIRVDebug::FlagBitField) Flags |= DINode::FlagBitField; if (Flags & DINode::FlagStaticMember && Ops.size() > MinOperandCount) { SPIRVValue *ConstVal = BM->get(Ops[ValueIdx]); assert(isConstantOpCode(ConstVal->getOpCode()) && "Static member must be a constant"); llvm::Value *Val = SPIRVReader->transValue(ConstVal, nullptr, nullptr); return getDIBuilder(DebugInst).createStaticMemberType( Scope, Name, File, LineNo, BaseType, Flags, cast(Val)); } const uint64_t Size = BM->get(Ops[SizeIdx])->getZExtIntValue(); const uint64_t Alignment = 0; return getDIBuilder(ParentInst) .createMemberType(Scope, Name, File, LineNo, Size, Alignment, OffsetInBits, Flags, BaseType); } DINode *SPIRVToLLVMDbgTran::transTypeEnum(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeEnum; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); uint64_t SizeInBits = BM->get(Ops[SizeIdx])->getZExtIntValue(); SPIRVWord AlignInBits = 0; SPIRVWord Flags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); if (Flags & SPIRVDebug::FlagIsFwdDecl) { return getDIBuilder(DebugInst).createForwardDecl( dwarf::DW_TAG_enumeration_type, Name, Scope, File, LineNo, AlignInBits, SizeInBits); } else { SmallVector Elts; for (size_t I = FirstEnumeratorIdx, E = Ops.size(); I < E; I += 2) { uint64_t Val = BM->get(Ops[I])->getZExtIntValue(); StringRef Name = getString(Ops[I + 1]); Elts.push_back(getDIBuilder(DebugInst).createEnumerator(Name, Val)); } DINodeArray Enumerators = getDIBuilder(DebugInst).getOrCreateArray(Elts); DIType *UnderlyingType = nullptr; SPIRVEntry *E = BM->getEntry(Ops[UnderlyingTypeIdx]); if (!isa(E)) UnderlyingType = transDebugInst(static_cast(E)); return getDIBuilder(DebugInst).createEnumerationType( Scope, Name, File, LineNo, SizeInBits, AlignInBits, Enumerators, UnderlyingType, "", Flags & SPIRVDebug::FlagIsEnumClass); } } DINode *SPIRVToLLVMDbgTran::transTypeFunction(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeFunction; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); DINode::DIFlags Flags = DINode::FlagZero; if (SPIRVFlags & SPIRVDebug::FlagIsLValueReference) Flags |= llvm::DINode::FlagLValueReference; if (SPIRVFlags & SPIRVDebug::FlagIsRValueReference) Flags |= llvm::DINode::FlagRValueReference; SPIRVEntry *E = BM->getEntry(Ops[ReturnTypeIdx]); MDNode *RT = isa(E) ? nullptr : transDebugInst(BM->get(Ops[ReturnTypeIdx])); SmallVector Elements{RT}; for (size_t I = FirstParameterIdx, E = Ops.size(); I < E; ++I) { SPIRVEntry *P = BM->getEntry(Ops[I]); MDNode *Param = isa(P) ? nullptr : transDebugInst(BM->get(Ops[I])); Elements.push_back(Param); } DITypeRefArray ArgTypes = getDIBuilder(DebugInst).getOrCreateTypeArray(Elements); return getDIBuilder(DebugInst).createSubroutineType(ArgTypes, Flags); } DINode * SPIRVToLLVMDbgTran::transTypePtrToMember(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypePtrToMember; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); SPIRVExtInst *Member = BM->get(Ops[MemberTypeIdx]); DIType *PointeeTy = transNonNullDebugType(Member); SPIRVExtInst *ContainingTy = BM->get(Ops[ParentIdx]); DIType *BaseTy = transNonNullDebugType(ContainingTy); return getDIBuilder(DebugInst).createMemberPointerType(PointeeTy, BaseTy, 0); } DINode *SPIRVToLLVMDbgTran::transLexicalBlock(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::LexicalBlock; const SPIRVWordVec &Ops = DebugInst->getArguments(); DIScope *ParentScope = getScope(BM->getEntry(Ops[ParentIdx])); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); if (Ops.size() > MinOperandCount) { StringRef Name = getString(Ops[NameIdx]); bool InlinedNamespace = false; if (DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { SPIRVValue *V = BM->get(Ops[InlineNamespaceIdx]); Value *Var = SPIRVReader->transValue(V, nullptr, nullptr); InlinedNamespace = cast(Var)->isOne(); } return getDIBuilder(DebugInst).createNameSpace(ParentScope, Name, InlinedNamespace); } unsigned Column = Ops[ColumnIdx]; return getDIBuilder(DebugInst).createLexicalBlock(ParentScope, File, LineNo, Column); } DINode *SPIRVToLLVMDbgTran::transLexicalBlockDiscriminator( const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::LexicalBlockDiscriminator; const SPIRVWordVec &Ops = DebugInst->getArguments(); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord Disc = getConstantValueOrLiteral(Ops, DiscriminatorIdx, DebugInst->getExtSetKind()); DIScope *ParentScope = getScope(BM->getEntry(Ops[ParentIdx])); return getDIBuilder(DebugInst).createLexicalBlockFile(ParentScope, File, Disc); } DINode *SPIRVToLLVMDbgTran::transFunction(const SPIRVExtInst *DebugInst, bool IsMainSubprogram) { using namespace SPIRVDebug::Operand::Function; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCountNonSem && "Invalid number of operands"); if (!isNonSemanticDebugInfo(DebugInst->getExtSetKind())) assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); DISubroutineType *Ty = transDebugInst(BM->get(Ops[TypeIdx])); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); StringRef LinkageName = getString(Ops[LinkageNameIdx]); SPIRVWord SPIRVDebugFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); DINode::DIFlags Flags = DINode::FlagZero; if (SPIRVDebugFlags & SPIRVDebug::FlagIsArtificial) Flags |= llvm::DINode::FlagArtificial; if (SPIRVDebugFlags & SPIRVDebug::FlagIsExplicit) Flags |= llvm::DINode::FlagExplicit; if (SPIRVDebugFlags & SPIRVDebug::FlagIsPrototyped) Flags |= llvm::DINode::FlagPrototyped; if (SPIRVDebugFlags & SPIRVDebug::FlagIsLValueReference) Flags |= llvm::DINode::FlagLValueReference; if (SPIRVDebugFlags & SPIRVDebug::FlagIsRValueReference) Flags |= llvm::DINode::FlagRValueReference; if ((SPIRVDebugFlags & SPIRVDebug::FlagAccess) == SPIRVDebug::FlagIsPublic) Flags |= llvm::DINode::FlagPublic; if (SPIRVDebugFlags & SPIRVDebug::FlagIsProtected) Flags |= llvm::DINode::FlagProtected; if (SPIRVDebugFlags & SPIRVDebug::FlagIsPrivate) Flags |= llvm::DINode::FlagPrivate; bool IsDefinition = SPIRVDebugFlags & SPIRVDebug::FlagIsDefinition; bool IsOptimized = SPIRVDebugFlags & SPIRVDebug::FlagIsOptimized; bool IsLocal = SPIRVDebugFlags & SPIRVDebug::FlagIsLocal; bool IsMainSubprogramFlag = IsMainSubprogram || (!isNonSemanticDebugInfo(DebugInst->getExtSetKind()) && BM->isEntryPoint(spv::ExecutionModelKernel, Ops[FunctionIdIdx])); DISubprogram::DISPFlags SPFlags = DISubprogram::toSPFlags( IsLocal, IsDefinition, IsOptimized, DISubprogram::SPFlagNonvirtual, IsMainSubprogramFlag); SPIRVWord ScopeLine = getConstantValueOrLiteral(Ops, ScopeLineIdx, DebugInst->getExtSetKind()); // Function declaration descriptor DISubprogram *FD = nullptr; if (isNonSemanticDebugInfo(DebugInst->getExtSetKind()) && Ops.size() > DeclarationNonSemIdx) { FD = transDebugInst( BM->get(Ops[DeclarationNonSemIdx])); } else if (Ops.size() > DeclarationIdx) { FD = transDebugInst( BM->get(Ops[DeclarationIdx])); } // Here we create fake array of template parameters. If it was plain nullptr, // the template parameter operand would be removed in DISubprogram::getImpl. // But we want it to be there, because if there is DebugTypeTemplate // instruction refering to this function, transTypeTemplate method must be // able to replace the template parameter operand, thus it must be in the // operands list. SmallVector Elts; DINodeArray TParams = getDIBuilder(DebugInst).getOrCreateArray(Elts); llvm::DITemplateParameterArray TParamsArray = TParams.get(); DISubprogram *DIS = nullptr; if (Scope && (isa(Scope) || isa(Scope)) && !IsDefinition) DIS = getDIBuilder(DebugInst).createMethod(Scope, Name, LinkageName, File, LineNo, Ty, 0, 0, nullptr, Flags, SPFlags, TParamsArray); else { // Create targetFuncName mostly for Fortran trampoline function if it is // the case StringRef TargetFunction = ""; if (DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200 && Ops.size() > TargetFunctionNameIdx) { TargetFunction = getString(Ops[TargetFunctionNameIdx]); } DIS = getDIBuilder(DebugInst).createFunction( Scope, Name, LinkageName, File, LineNo, Ty, ScopeLine, Flags, SPFlags, TParamsArray, FD, /*ThrownTypes*/ nullptr, /*Annotations*/ nullptr, TargetFunction); } DebugInstCache[DebugInst] = DIS; // At this point, we don't have info about the function definition for // NonSemantic.Shader debug info. If function definition is present, it'll be // translated later within the function scope. // For "default" debug info we do translate function body here. if (!isNonSemanticDebugInfo(DebugInst->getExtSetKind())) transFunctionBody(DIS, Ops[FunctionIdIdx]); return DIS; } void SPIRVToLLVMDbgTran::transFunctionBody(DISubprogram *DIS, SPIRVId FuncId) { FuncMap[FuncId] = DIS; SPIRVEntry *E = BM->getEntry(FuncId); if (E->getOpCode() == OpFunction) { SPIRVFunction *BF = static_cast(E); llvm::Function *F = SPIRVReader->transFunction(BF); assert(F && "Translation of function failed!"); if (!F->hasMetadata("dbg")) F->setMetadata("dbg", DIS); } } DINode * SPIRVToLLVMDbgTran::transFunctionDefinition(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::FunctionDefinition; const SPIRVWordVec &Ops = DebugInst->getArguments(); SPIRVExtInst *Func = BM->get(Ops[FunctionIdx]); DISubprogram *LLVMFunc = cast(DebugInstCache[Func]); transFunctionBody(LLVMFunc, Ops[DefinitionIdx]); return nullptr; } DINode *SPIRVToLLVMDbgTran::transFunctionDecl(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::FunctionDeclaration; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() == OperandCount && "Invalid number of operands"); // Scope DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); StringRef Name = getString(Ops[NameIdx]); StringRef LinkageName = getString(Ops[LinkageNameIdx]); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DISubroutineType *Ty = transDebugInst(BM->get(Ops[TypeIdx])); SPIRVWord SPIRVDebugFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); bool IsDefinition = SPIRVDebugFlags & SPIRVDebug::FlagIsDefinition; bool IsOptimized = SPIRVDebugFlags & SPIRVDebug::FlagIsOptimized; bool IsLocal = SPIRVDebugFlags & SPIRVDebug::FlagIsLocal; DINode::DIFlags Flags = DINode::FlagZero; if (SPIRVDebugFlags & SPIRVDebug::FlagIsArtificial) Flags |= llvm::DINode::FlagArtificial; if (SPIRVDebugFlags & SPIRVDebug::FlagIsExplicit) Flags |= llvm::DINode::FlagExplicit; if (SPIRVDebugFlags & SPIRVDebug::FlagIsPrototyped) Flags |= llvm::DINode::FlagPrototyped; if (SPIRVDebugFlags & SPIRVDebug::FlagIsLValueReference) Flags |= llvm::DINode::FlagLValueReference; if (SPIRVDebugFlags & SPIRVDebug::FlagIsRValueReference) Flags |= llvm::DINode::FlagRValueReference; if ((SPIRVDebugFlags & SPIRVDebug::FlagAccess) == SPIRVDebug::FlagIsPublic) Flags |= llvm::DINode::FlagPublic; if (SPIRVDebugFlags & SPIRVDebug::FlagIsProtected) Flags |= llvm::DINode::FlagProtected; if (SPIRVDebugFlags & SPIRVDebug::FlagIsPrivate) Flags |= llvm::DINode::FlagPrivate; // Here we create fake array of template parameters. If it was plain nullptr, // the template parameter operand would be removed in DISubprogram::getImpl. // But we want it to be there, because if there is DebugTypeTemplate // instruction refering to this function, transTypeTemplate method must be // able to replace the template parameter operand, thus it must be in the // operands list. SmallVector Elts; DINodeArray TParams = getDIBuilder(DebugInst).getOrCreateArray(Elts); llvm::DITemplateParameterArray TParamsArray = TParams.get(); DISubprogram *DIS = nullptr; DISubprogram::DISPFlags SPFlags = DISubprogram::toSPFlags(IsLocal, IsDefinition, IsOptimized); if (isa(Scope) || isa(Scope)) DIS = getDIBuilder(DebugInst).createMethod(Scope, Name, LinkageName, File, LineNo, Ty, 0, 0, nullptr, Flags, SPFlags, TParamsArray); else { // Since a function declaration doesn't have any retained nodes, resolve // the temporary placeholder for them immediately. DIS = getDIBuilder(DebugInst).createTempFunctionFwdDecl( Scope, Name, LinkageName, File, LineNo, Ty, 0, Flags, SPFlags, TParamsArray); llvm::TempMDNode FwdDecl(cast(DIS)); DIS = getDIBuilder(DebugInst).replaceTemporary(std::move(FwdDecl), DIS); } DebugInstCache[DebugInst] = DIS; return DIS; } MDNode *SPIRVToLLVMDbgTran::transEntryPoint(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::EntryPoint; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() == OperandCount && "Invalid number of operands"); SPIRVExtInst *EP = BM->get(Ops[EntryPointIdx]); SPIRVExtInst *CU = BM->get(Ops[CompilationUnitIdx]); std::string Producer = getString(Ops[CompilerSignatureIdx]); std::string CLArgs = getString(Ops[CommandLineArgsIdx]); DICompileUnit *C = transCompilationUnit(CU, Producer, CLArgs); // NOLINT DebugInstCache[CU] = C; return transFunction(EP, true /*IsMainSubprogram*/); } MDNode *SPIRVToLLVMDbgTran::transGlobalVariable(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::GlobalVariable; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); DIType *Ty = transNonNullDebugType(BM->get(Ops[TypeIdx])); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIScope *Parent = getScope(BM->getEntry(Ops[ParentIdx])); StringRef LinkageName = getString(Ops[LinkageNameIdx]); DIDerivedType *StaticMemberDecl = nullptr; if (Ops.size() > MinOperandCount) { StaticMemberDecl = transDebugInst( BM->get(Ops[StaticMemberDeclarationIdx])); } DIExpression *DIExpr = nullptr; // Check if Ops[VariableIdx] is not being used to hold a variable operand. // Instead it is being used to hold an Expression that holds the initial // value of the GlobalVariable. if (getDbgInst(Ops[VariableIdx])) DIExpr = transDebugInst(BM->get(Ops[VariableIdx])); SPIRVWord Flags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); bool IsLocal = Flags & SPIRVDebug::FlagIsLocal; bool IsDefinition = Flags & SPIRVDebug::FlagIsDefinition; MDNode *VarDecl = nullptr; if (IsDefinition) { VarDecl = getDIBuilder(DebugInst).createGlobalVariableExpression( Parent, Name, LinkageName, File, LineNo, Ty, IsLocal, IsDefinition, DIExpr, StaticMemberDecl); } else { VarDecl = getDIBuilder(DebugInst).createTempGlobalVariableFwdDecl( Parent, Name, LinkageName, File, LineNo, Ty, IsLocal, StaticMemberDecl); // replaceAllUsesWith call makes VarDecl non-temp. // Otherwise DIBuilder will crash at finalization. llvm::TempMDNode TMP(VarDecl); VarDecl = getDIBuilder(DebugInst).replaceTemporary(std::move(TMP), VarDecl); } // Ops[VariableIdx] was not used to hold an Expression with the initial value // for the GlobalVariable if (!DIExpr) { // If the variable has no initializer Ops[VariableIdx] is OpDebugInfoNone. // Otherwise Ops[VariableIdx] may be a global variable or a constant(C++ // static const). if (VarDecl && !getDbgInst(Ops[VariableIdx])) { SPIRVValue *V = BM->get(Ops[VariableIdx]); Value *Var = SPIRVReader->transValue(V, nullptr, nullptr); llvm::GlobalVariable *GV = dyn_cast_or_null(Var); if (GV && !GV->hasMetadata("dbg")) GV->addMetadata("dbg", *VarDecl); } } return VarDecl; } DINode *SPIRVToLLVMDbgTran::transLocalVariable(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::LocalVariable; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); StringRef Name = getString(Ops[NameIdx]); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIType *Ty = transNonNullDebugType(BM->get(Ops[TypeIdx])); DINode::DIFlags Flags = DINode::FlagZero; SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); if (SPIRVFlags & SPIRVDebug::FlagIsArtificial) Flags |= DINode::FlagArtificial; if (SPIRVFlags & SPIRVDebug::FlagIsObjectPointer) Flags |= DINode::FlagObjectPointer; if (Ops.size() > ArgNumberIdx) return getDIBuilder(DebugInst).createParameterVariable( Scope, Name, Ops[ArgNumberIdx], File, LineNo, Ty, true, Flags); return getDIBuilder(DebugInst).createAutoVariable(Scope, Name, File, LineNo, Ty, true, Flags); } DINode *SPIRVToLLVMDbgTran::transTypedef(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::Typedef; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); DIFile *File = getFile(Ops[SourceIdx]); SPIRVWord LineNo = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); StringRef Alias = getString(Ops[NameIdx]); SPIRVEntry *TypeInst = BM->getEntry(Ops[BaseTypeIdx]); DIType *Ty = transDebugInst(static_cast(TypeInst)); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); assert(Scope && "Typedef should have a parent scope"); return getDIBuilder(DebugInst).createTypedef(Ty, Alias, File, LineNo, Scope); } DINode *SPIRVToLLVMDbgTran::transTypeInheritance(const SPIRVExtInst *DebugInst, DIType *ChildClass) { using namespace SPIRVDebug::Operand::TypeInheritance; // The value is used when assertions are enabled [[maybe_unused]] unsigned OperandCount; unsigned ParentIdx, OffsetIdx, FlagsIdx; if (isNonSemanticDebugInfo(DebugInst->getExtSetKind())) { if (!ChildClass) { // Will be translated later when processing TypeMember's parent return nullptr; } OperandCount = NonSemantic::OperandCount; ParentIdx = NonSemantic::ParentIdx; OffsetIdx = NonSemantic::OffsetIdx; FlagsIdx = NonSemantic::FlagsIdx; } else { OperandCount = NonSemantic::OperandCount; ParentIdx = OpenCL::ParentIdx; OffsetIdx = OpenCL::OffsetIdx; FlagsIdx = OpenCL::FlagsIdx; } const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); DIType *Parent = transDebugInst(BM->get(Ops[ParentIdx])); DINode::DIFlags Flags = DINode::FlagZero; SPIRVWord SPIRVFlags = getConstantValueOrLiteral(Ops, FlagsIdx, DebugInst->getExtSetKind()); if ((SPIRVFlags & SPIRVDebug::FlagAccess) == SPIRVDebug::FlagIsPublic) Flags |= llvm::DINode::FlagPublic; if ((SPIRVFlags & SPIRVDebug::FlagAccess) == SPIRVDebug::FlagIsProtected) Flags |= llvm::DINode::FlagProtected; if ((SPIRVFlags & SPIRVDebug::FlagAccess) == SPIRVDebug::FlagIsPrivate) Flags |= llvm::DINode::FlagPrivate; uint64_t OffsetVal = BM->get(Ops[OffsetIdx])->getZExtIntValue(); DIType *Child; if (isNonSemanticDebugInfo(DebugInst->getExtSetKind())) { Child = ChildClass; } else { Child = transDebugInst(BM->get(Ops[OpenCL::ChildIdx])); } return getDIBuilder(DebugInst).createInheritance(Child, Parent, OffsetVal, 0, Flags); } DINode * SPIRVToLLVMDbgTran::transTypeTemplateParameter(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeTemplateParameter; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); SPIRVEntry *ActualType = BM->getEntry(Ops[TypeIdx]); DIType *Ty = nullptr; if (!isa(ActualType)) Ty = transDebugInst(static_cast(ActualType)); DIScope *Context = nullptr; if (!getDbgInst(Ops[ValueIdx])) { SPIRVValue *Val = BM->get(Ops[ValueIdx]); Value *V = SPIRVReader->transValue(Val, nullptr, nullptr); return getDIBuilder(DebugInst).createTemplateValueParameter( Context, Name, Ty, false, cast(V)); } return getDIBuilder(DebugInst).createTemplateTypeParameter(Context, Name, Ty, false); } DINode *SPIRVToLLVMDbgTran::transTypeTemplateTemplateParameter( const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeTemplateTemplateParameter; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); StringRef TemplName = getString(Ops[TemplateNameIdx]); DIScope *Context = nullptr; return getDIBuilder(DebugInst).createTemplateTemplateParameter( Context, Name, nullptr, TemplName); } DINode *SPIRVToLLVMDbgTran::transTypeTemplateParameterPack( const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeTemplateParameterPack; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); StringRef Name = getString(Ops[NameIdx]); SmallVector Elts; for (size_t I = FirstParameterIdx, E = Ops.size(); I < E; ++I) { Elts.push_back(transDebugInst(BM->get(Ops[I]))); } DINodeArray Pack = getDIBuilder(DebugInst).getOrCreateArray(Elts); DIScope *Context = nullptr; return getDIBuilder(DebugInst).createTemplateParameterPack(Context, Name, nullptr, Pack); } MDNode *SPIRVToLLVMDbgTran::transTypeTemplate(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::TypeTemplate; const SPIRVWordVec &Ops = DebugInst->getArguments(); const size_t NumOps = Ops.size(); assert(NumOps >= MinOperandCount && "Invalid number of operands"); auto *Templ = BM->get(Ops[TargetIdx]); MDNode *D = transDebugInst(Templ); SmallVector Elts; for (size_t I = FirstParameterIdx; I < NumOps; ++I) { Elts.push_back(transDebugInst(BM->get(Ops[I]))); } DINodeArray TParams = getDIBuilder(DebugInst).getOrCreateArray(Elts); if (DICompositeType *Comp = dyn_cast(D)) { getDIBuilder(DebugInst).replaceArrays(Comp, Comp->getElements(), TParams); return Comp; } if (isa(D)) { // This constant matches with one used in // DISubprogram::getRawTemplateParams() const unsigned TemplateParamsIndex = 9; D->replaceOperandWith(TemplateParamsIndex, TParams.get()); return D; } llvm_unreachable("Invalid template"); } DINode *SPIRVToLLVMDbgTran::transImportedEntry(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::ImportedEntity; const SPIRVWordVec &Ops = DebugInst->getArguments(); // FIXME: 'OpenCL/bugged' version is kept because it's hard to remove it // It's W/A for missing 2nd index in OpenCL's implementation const SPIRVWord OffsetIdx = static_cast(isNonSemanticDebugInfo(DebugInst->getExtSetKind())); assert(Ops.size() == (OpenCL::OperandCount - OffsetIdx) && "Invalid number of operands"); DIScope *Scope = getScope(BM->getEntry(Ops[OpenCL::ParentIdx - OffsetIdx])); const SPIRVWord Line = getConstantValueOrLiteral( Ops, OpenCL::LineIdx - OffsetIdx, DebugInst->getExtSetKind()); DIFile *File = getFile(Ops[OpenCL::SourceIdx - OffsetIdx]); auto *Entity = transDebugInst( BM->get(Ops[OpenCL::EntityIdx - OffsetIdx])); const SPIRVWord Tag = getConstantValueOrLiteral(Ops, OpenCL::TagIdx, DebugInst->getExtSetKind()); if (Tag == SPIRVDebug::ImportedModule) { if (!Entity) return getDIBuilder(DebugInst).createImportedModule( Scope, static_cast(nullptr), File, Line); if (DIModule *DM = dyn_cast(Entity)) return getDIBuilder(DebugInst).createImportedModule(Scope, DM, File, Line); if (DIImportedEntity *IE = dyn_cast(Entity)) return getDIBuilder(DebugInst).createImportedModule(Scope, IE, File, Line); if (DINamespace *NS = dyn_cast(Entity)) return getDIBuilder(DebugInst).createImportedModule(Scope, NS, File, Line); } if (Tag == SPIRVDebug::ImportedDeclaration) { const StringRef Name = getString(Ops[OpenCL::NameIdx]); if (DIGlobalVariableExpression *GVE = dyn_cast(Entity)) return getDIBuilder(DebugInst).createImportedDeclaration( Scope, GVE->getVariable(), File, Line, Name); return getDIBuilder(DebugInst).createImportedDeclaration(Scope, Entity, File, Line, Name); } llvm_unreachable("Unexpected kind of imported entity!"); } DINode *SPIRVToLLVMDbgTran::transModule(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::ModuleINTEL; const SPIRVWordVec &Ops = DebugInst->getArguments(); assert(Ops.size() >= OperandCount && "Invalid number of operands"); DIScope *Scope = getScope(BM->getEntry(Ops[ParentIdx])); SPIRVWord Line = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); DIFile *File = getFile(Ops[SourceIdx]); StringRef Name = getString(Ops[NameIdx]); StringRef ConfigMacros = getString(Ops[ConfigMacrosIdx]); StringRef IncludePath = getString(Ops[IncludePathIdx]); StringRef ApiNotes = getString(Ops[ApiNotesIdx]); bool IsDecl = getConstantValueOrLiteral(Ops, IsDeclIdx, DebugInst->getExtSetKind()); return getDIBuilder(DebugInst).createModule( Scope, Name, ConfigMacros, IncludePath, ApiNotes, File, Line, IsDecl); } MDNode *SPIRVToLLVMDbgTran::transExpression(const SPIRVExtInst *DebugInst) { const SPIRVWordVec &Args = DebugInst->getArguments(); std::vector Ops; for (SPIRVId A : Args) { using namespace SPIRVDebug::Operand::Operation; SPIRVExtInst *O = BM->get(A); const SPIRVWordVec &Operands = O->getArguments(); auto OpCode = static_cast(getConstantValueOrLiteral( Operands, OpCodeIdx, DebugInst->getExtSetKind())); Ops.push_back(SPIRV::DbgExpressionOpCodeMap::rmap(OpCode)); for (unsigned I = 1, E = Operands.size(); I < E; ++I) { Ops.push_back( getConstantValueOrLiteral(Operands, I, DebugInst->getExtSetKind())); } } ArrayRef Addr(Ops.data(), Ops.size()); return getDIBuilder(DebugInst).createExpression(Addr); } DIType * SPIRVToLLVMDbgTran::transNonNullDebugType(const SPIRVExtInst *DebugInst) { if (DebugInst->getExtOp() != SPIRVDebug::DebugInfoNone) return transDebugInst(DebugInst); return getDIBuilder(DebugInst).createUnspecifiedType("SPIRV unknown type"); } MDNode *SPIRVToLLVMDbgTran::transDebugInstImpl(const SPIRVExtInst *DebugInst) { switch (DebugInst->getExtOp()) { case SPIRVDebug::DebugInfoNone: return nullptr; case SPIRVDebug::CompilationUnit: return transCompilationUnit(DebugInst); case SPIRVDebug::TypeBasic: return transTypeBasic(DebugInst); case SPIRVDebug::TypeQualifier: return transTypeQualifier(DebugInst); case SPIRVDebug::TypePointer: return transTypePointer(DebugInst); case SPIRVDebug::TypeArray: return transTypeArray(DebugInst); case SPIRVDebug::TypeSubrange: return transTypeSubrange(DebugInst); case SPIRVDebug::TypeString: return transTypeString(DebugInst); case SPIRVDebug::TypeVector: return transTypeVector(DebugInst); case SPIRVDebug::TypeComposite: return transTypeComposite(DebugInst); case SPIRVDebug::TypeMember: return transTypeMember(DebugInst); case SPIRVDebug::TypePtrToMember: return transTypePtrToMember(DebugInst); case SPIRVDebug::TypeEnum: return transTypeEnum(DebugInst); case SPIRVDebug::TypeFunction: return transTypeFunction(DebugInst); case SPIRVDebug::LexicalBlock: return transLexicalBlock(DebugInst); case SPIRVDebug::LexicalBlockDiscriminator: return transLexicalBlockDiscriminator(DebugInst); case SPIRVDebug::Function: return transFunction(DebugInst); case SPIRVDebug::FunctionDeclaration: return transFunctionDecl(DebugInst); case SPIRVDebug::FunctionDefinition: return transFunctionDefinition(DebugInst); case SPIRVDebug::EntryPoint: return transEntryPoint(DebugInst); case SPIRVDebug::GlobalVariable: return transGlobalVariable(DebugInst); case SPIRVDebug::LocalVariable: return transLocalVariable(DebugInst); case SPIRVDebug::Typedef: return transTypedef(DebugInst); case SPIRVDebug::InlinedAt: return transDebugInlined(DebugInst); case SPIRVDebug::TypeInheritance: return transTypeInheritance(DebugInst); case SPIRVDebug::TypeTemplateParameter: return transTypeTemplateParameter(DebugInst); case SPIRVDebug::TypeTemplateTemplateParameter: return transTypeTemplateTemplateParameter(DebugInst); case SPIRVDebug::TypeTemplateParameterPack: return transTypeTemplateParameterPack(DebugInst); case SPIRVDebug::TypeTemplate: return transTypeTemplate(DebugInst); case SPIRVDebug::ImportedEntity: return transImportedEntry(DebugInst); case SPIRVDebug::Module: case SPIRVDebug::ModuleINTEL: return transModule(DebugInst); case SPIRVDebug::Operation: // To be translated with transExpression case SPIRVDebug::Source: // To be used by other instructions case SPIRVDebug::SourceContinued: case SPIRVDebug::BuildIdentifier: // To be used by transCompilationUnit case SPIRVDebug::StoragePath: // To be used by transCompilationUnit return nullptr; case SPIRVDebug::Expression: return transExpression(DebugInst); case SPIRVDebug::TypeArrayDynamic: return transTypeArrayDynamic(DebugInst); default: llvm_unreachable("Not implemented SPIR-V debug instruction!"); } } Instruction * SPIRVToLLVMDbgTran::transDebugIntrinsic(const SPIRVExtInst *DebugInst, BasicBlock *BB) { auto GetLocalVar = [&](SPIRVId Id) -> std::pair { auto *LV = transDebugInst(BM->get(Id)); DebugLoc DL = DILocation::get(M->getContext(), LV->getLine(), /*Column=*/0, LV->getScope()); return std::make_pair(LV, DL); }; auto GetValue = [&](SPIRVId Id) -> Value * { auto *V = BM->get(Id); return SPIRVReader->transValue(V, BB->getParent(), BB); }; auto GetExpression = [&](SPIRVId Id) -> DIExpression * { return transDebugInst(BM->get(Id)); }; SPIRVWordVec Ops = DebugInst->getArguments(); switch (DebugInst->getExtOp()) { case SPIRVDebug::Scope: case SPIRVDebug::NoScope: case SPIRVDebug::FunctionDefinition: return nullptr; case SPIRVDebug::Declare: { using namespace SPIRVDebug::Operand::DebugDeclare; auto LocalVar = GetLocalVar(Ops[DebugLocalVarIdx]); if (getDbgInst(Ops[VariableIdx])) { // If we don't have the variable(e.g. alloca might be promoted by mem2reg) // we should generate the following IR: // call void @llvm.dbg.declare(metadata !4, metadata !14, metadata !5) // !4 = !{} // DIBuilder::insertDeclare doesn't allow to pass nullptr for the Storage // parameter. To work around this limitation we create a dummy temp // alloca, use it to create llvm.dbg.declare, and then remove the alloca. auto *AI = new AllocaInst(Type::getInt8Ty(M->getContext()), 0, "tmp", BB); auto *DbgDeclare = getDIBuilder(DebugInst).insertDeclare( AI, LocalVar.first, GetExpression(Ops[ExpressionIdx]), LocalVar.second, BB); AI->eraseFromParent(); return DbgDeclare; } return getDIBuilder(DebugInst).insertDeclare( GetValue(Ops[VariableIdx]), LocalVar.first, GetExpression(Ops[ExpressionIdx]), LocalVar.second, BB); } case SPIRVDebug::Value: { using namespace SPIRVDebug::Operand::DebugValue; auto LocalVar = GetLocalVar(Ops[DebugLocalVarIdx]); Value *Val = GetValue(Ops[ValueIdx]); DIExpression *Expr = GetExpression(Ops[ExpressionIdx]); auto *DbgValIntr = getDIBuilder(DebugInst).insertDbgValueIntrinsic( Val, LocalVar.first, Expr, LocalVar.second, BB); if (Expr->getNumLocationOperands() == 1) { SmallVector MDs = {ValueAsMetadata::get(Val)}; DIArgList *AL = DIArgList::get(M->getContext(), MDs); cast(DbgValIntr)->setRawLocation(AL); } return DbgValIntr; } default: llvm_unreachable("Unknown debug intrinsic!"); } } DebugLoc SPIRVToLLVMDbgTran::transDebugScope(const SPIRVInstruction *Inst) { unsigned Line = 0; unsigned Col = 0; MDNode *Scope = nullptr; MDNode *InlinedAt = nullptr; // If DebugLine and OpLine are both active give DebugLine priority if (auto DL = Inst->getDebugLine()) { using namespace SPIRVDebug::Operand::DebugLine; const SPIRVWordVec DebugLineArgs = DL->getArguments(); Line = getConstantValueOrLiteral(DebugLineArgs, StartIdx, DL->getExtSetKind()); Col = getConstantValueOrLiteral(DebugLineArgs, ColumnStartIdx, DL->getExtSetKind()); } else if (auto L = Inst->getLine()) { Line = L->getLine(); Col = L->getColumn(); } if (SPIRVEntry *S = Inst->getDebugScope()) { using namespace SPIRVDebug::Operand::Scope; SPIRVExtInst *DbgScope = static_cast(S); SPIRVWordVec Ops = DbgScope->getArguments(); Scope = getScope(BM->getEntry(Ops[ScopeIdx])); if (Ops.size() > InlinedAtIdx) InlinedAt = transDebugInst(BM->get(Ops[InlinedAtIdx])); return DILocation::get(M->getContext(), Line, Col, Scope, InlinedAt); } return DebugLoc(); } MDNode *SPIRVToLLVMDbgTran::transDebugInlined(const SPIRVExtInst *DebugInst) { // There is a Column operand in NonSemantic.Shader.200 spec if (DebugInst->getExtSetKind() == SPIRV::SPIRVEIS_NonSemantic_Shader_DebugInfo_200) return transDebugInlinedNonSemanticShader200(DebugInst); using namespace SPIRVDebug::Operand::InlinedAt::OpenCL; SPIRVWordVec Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); SPIRVWord Line = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); unsigned Col = 0; // DebugInlinedAt instruction has no column operand DILocalScope *Scope = cast(getScope(BM->getEntry(Ops[ScopeIdx]))); DILocation *InlinedAt = nullptr; if (Ops.size() > InlinedIdx) { InlinedAt = transDebugInst(BM->get(Ops[InlinedIdx])); } return DILocation::getDistinct(M->getContext(), Line, Col, Scope, InlinedAt); } MDNode *SPIRVToLLVMDbgTran::transDebugInlinedNonSemanticShader200( const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::InlinedAt::NonSemantic; SPIRVWordVec Ops = DebugInst->getArguments(); assert(Ops.size() >= MinOperandCount && "Invalid number of operands"); const SPIRVWord Line = getConstantValueOrLiteral(Ops, LineIdx, DebugInst->getExtSetKind()); const unsigned Col = getConstantValueOrLiteral(Ops, ColumnIdx, DebugInst->getExtSetKind()); DILocalScope *Scope = cast(getScope(BM->getEntry(Ops[ScopeIdx]))); DILocation *InlinedAt = nullptr; if (Ops.size() > InlinedIdx) { InlinedAt = transDebugInst(BM->get(Ops[InlinedIdx])); } return DILocation::getDistinct(M->getContext(), Line, Col, Scope, InlinedAt); } void SPIRVToLLVMDbgTran::finalize() { if (!Enable) return; for (const auto &Builder : BuilderMap) Builder.second->finalize(); } DIFile *SPIRVToLLVMDbgTran::getFile(const SPIRVId SourceId) { using namespace SPIRVDebug::Operand::Source; SPIRVExtInst *Source = BM->get(SourceId); assert(Source->getExtOp() == SPIRVDebug::Source && "DebugSource instruction is expected"); SPIRVWordVec SourceArgs = Source->getArguments(); assert(SourceArgs.size() >= MinOperandCount && "Invalid number of operands"); if (SourceArgs.size() == MinOperandCount) return getDIFile(getString(SourceArgs[FileIdx])); if (!isNonSemanticDebugInfo(Source->getExtSetKind())) { const std::string ChecksumStr = getDbgInst(SourceArgs[TextIdx]) ? "" : getString(SourceArgs[TextIdx]); return getDIFile(getString(SourceArgs[FileIdx]), ParseChecksum(ChecksumStr)); } Optional> CS; SPIRVWord StrIdx = SourceArgs[TextIdx]; if (Source->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { if (SourceArgs.size() >= MaxOperandCount - 1) { // 2 optional parameters are ChecksumKind and ChecksumValue - they should // go together if (!getDbgInst(SourceArgs[ChecksumKind]) && !getDbgInst(SourceArgs[ChecksumValue])) { const llvm::DIFile::ChecksumKind Kind = SPIRV::DbgChecksumKindMap::rmap( static_cast( BM->get(SourceArgs[ChecksumKind]) ->getZExtIntValue())); const StringRef Checksum = getString(SourceArgs[ChecksumValue]); const size_t ChecksumEndPos = Checksum.find_if_not(llvm::isHexDigit); CS.emplace(Kind, Checksum.substr(0, ChecksumEndPos)); } } // Among optional parameters - text is always the last one (either 1st or // 3rd) if (SourceArgs.size() == MaxOperandCount || SourceArgs.size() == MaxOperandCount - 2) StrIdx = SourceArgs[TextNonSemIdx]; else StrIdx = SPIRVID_INVALID; } return getDIFile(getString(SourceArgs[FileIdx]), CS, getStringSourceContinued(StrIdx, Source)); } void SPIRVToLLVMDbgTran::setBuildIdentifierAndStoragePath() { #ifndef NDEBUG bool FoundBuildIdentifier{false}; bool FoundStoragePath{false}; #endif for (SPIRVExtInst *EI : BM->getDebugInstVec()) { if (EI->getExtOp() == SPIRVDebug::BuildIdentifier) { using namespace SPIRVDebug::Operand::BuildIdentifier; SPIRVWordVec BuildIdentifierArgs = EI->getArguments(); assert(BuildIdentifierArgs.size() == OperandCount && "Invalid number of operands"); assert(!FoundBuildIdentifier && "More than one BuildIdentifier instruction not allowed"); BuildIdentifier = strtoull( getString(BuildIdentifierArgs[IdentifierIdx]).c_str(), NULL, 10); #ifndef NDEBUG FoundBuildIdentifier = true; #endif } else if (EI->getExtOp() == SPIRVDebug::StoragePath) { using namespace SPIRVDebug::Operand::StoragePath; SPIRVWordVec StoragePathArgs = EI->getArguments(); assert(StoragePathArgs.size() == OperandCount && "Invalid number of operands"); assert(!FoundStoragePath && "More than one StoragePath instruction not allowed"); StoragePath = getString(StoragePathArgs[PathIdx]); #ifndef NDEBUG FoundStoragePath = true; #endif } } assert(((FoundBuildIdentifier && FoundStoragePath) || (!FoundBuildIdentifier && !FoundStoragePath)) && "BuildIdentifier and StoragePath must both be set or both unset"); } DIBuilder &SPIRVToLLVMDbgTran::getDIBuilder(const SPIRVExtInst *DebugInst) { assert(BuilderMap.size() != 0 && "No debug compile units"); if (BuilderMap.size() == 1) return *BuilderMap.begin()->second; while (DebugInst->getExtOp() != SPIRVDebug::CompilationUnit) { if (DebugInst->getExtOp() == SPIRVDebug::DebugInfoNone) return *BuilderMap.begin()->second; const SPIRVWordVec &Ops = DebugInst->getArguments(); SPIRVWord ParentScopeIdx = 0; if (!hasDbgInstParentScopeIdx(DebugInst->getExtOp(), ParentScopeIdx, DebugInst->getExtSetKind())) return *BuilderMap.begin()->second; if (SPIRVEntry *Scope = BM->getEntry(Ops[ParentScopeIdx])) { DebugInst = static_cast(Scope); continue; } return *BuilderMap.begin()->second; } return *BuilderMap[DebugInst->getId()]; } SPIRVToLLVMDbgTran::SplitFileName::SplitFileName(const string &FileName) { auto Loc = FileName.find_last_of("/\\"); if (Loc != std::string::npos) { BaseName = FileName.substr(Loc + 1); Path = FileName.substr(0, Loc); } else { BaseName = FileName; Path = "."; } } std::string SPIRVToLLVMDbgTran::findModuleProducer() { for (const auto &I : BM->getModuleProcessedVec()) { if (I->getProcessStr().find(SPIRVDebug::ProducerPrefix) != std::string::npos) { return I->getProcessStr().substr(SPIRVDebug::ProducerPrefix.size()); } } return "spirv"; } Optional> SPIRVToLLVMDbgTran::ParseChecksum(StringRef Text) { // Example of "Text" variable: // "SomeInfo//__CSK_MD5:7bb56387968a9caa6e9e35fff94eaf7b:OtherInfo" Optional> CS; auto KindPos = Text.find(SPIRVDebug::ChecksumKindPrefx); if (KindPos != StringRef::npos) { auto ColonPos = Text.find(":", KindPos); KindPos += string("//__").size(); auto KindStr = Text.substr(KindPos, ColonPos - KindPos); auto Checksum = Text.substr(ColonPos).ltrim(':'); if (auto Kind = DIFile::getChecksumKind(KindStr)) { size_t ChecksumEndPos = Checksum.find_if_not(llvm::isHexDigit); CS.emplace(Kind.getValue(), Checksum.substr(0, ChecksumEndPos)); } } return CS; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToLLVMDbgTran.h000066400000000000000000000212251506646577400225020ustar00rootroot00000000000000//===- SPIRVToLLVMDbgTran.h - Converts SPIR-V DebugInfo to LLVM -*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2018 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements translation of debug info from SPIR-V to LLVM metadata // //===----------------------------------------------------------------------===// #ifndef SPIRVTOLLVMDBGTRAN_H #define SPIRVTOLLVMDBGTRAN_H #include "SPIRVInstruction.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugLoc.h" #include namespace llvm { class Module; class Value; class Instruction; class Type; } // namespace llvm using namespace llvm; namespace SPIRV { class SPIRVToLLVM; class SPIRVEntry; class SPIRVFunction; class SPIRVValue; class SPIRVToLLVMDbgTran { public: typedef std::vector SPIRVWordVec; SPIRVToLLVMDbgTran(SPIRVModule *TBM, Module *TM, SPIRVToLLVM *Reader); void addDbgInfoVersion(); void transDbgInfo(const SPIRVValue *SV, Value *V); template T *transDebugInst(const SPIRVExtInst *DebugInst) { assert((DebugInst->getExtSetKind() == SPIRVEIS_Debug || DebugInst->getExtSetKind() == SPIRVEIS_OpenCL_DebugInfo_100 || DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || DebugInst->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) && "Unexpected extended instruction set"); auto It = DebugInstCache.find(DebugInst); if (It != DebugInstCache.end()) return static_cast(It->second); MDNode *Res = transDebugInstImpl(DebugInst); DebugInstCache[DebugInst] = Res; return static_cast(Res); } Instruction *transDebugIntrinsic(const SPIRVExtInst *DebugInst, BasicBlock *BB); void finalize(); private: DIFile *getFile(const SPIRVId SourceId); DIFile *getDIFile(const std::string &FileName, Optional> CS = None, Optional Source = None); DIFile *getDIFile(const SPIRVEntry *E); unsigned getLineNo(const SPIRVEntry *E); MDNode *transDebugInstImpl(const SPIRVExtInst *DebugInst); DIType *transNonNullDebugType(const SPIRVExtInst *DebugInst); llvm::DebugLoc transDebugLocation(const SPIRVExtInst *DebugInst); llvm::DebugLoc transDebugScope(const SPIRVInstruction *Inst); MDNode *transDebugInlined(const SPIRVExtInst *Inst); MDNode *transDebugInlinedNonSemanticShader200(const SPIRVExtInst *Inst); void appendToSourceLangLiteral(DICompileUnit *CompileUnit, SPIRVWord SourceLang); DICompileUnit *transCompilationUnit(const SPIRVExtInst *DebugInst, const std::string CompilerVersion = "", const std::string Flags = ""); DIBasicType *transTypeBasic(const SPIRVExtInst *DebugInst); DIDerivedType *transTypeQualifier(const SPIRVExtInst *DebugInst); DIType *transTypePointer(const SPIRVExtInst *DebugInst); DICompositeType *transTypeArray(const SPIRVExtInst *DebugInst); DICompositeType *transTypeArrayOpenCL(const SPIRVExtInst *DebugInst); DICompositeType *transTypeArrayNonSemantic(const SPIRVExtInst *DebugInst); DICompositeType *transTypeArrayDynamic(const SPIRVExtInst *DebugInst); DICompositeType *transTypeVector(const SPIRVExtInst *DebugInst); DICompositeType *transTypeComposite(const SPIRVExtInst *DebugInst); DISubrange *transTypeSubrange(const SPIRVExtInst *DebugInst); DIStringType *transTypeString(const SPIRVExtInst *DebugInst); DINode *transTypeMember(const SPIRVExtInst *DebugInst, const SPIRVExtInst *ParentInst = nullptr, DIScope *Scope = nullptr); DINode *transTypeMemberOpenCL(const SPIRVExtInst *DebugInst); DINode *transTypeMemberNonSemantic(const SPIRVExtInst *DebugInst, const SPIRVExtInst *ParentInst, DIScope *Scope); DINode *transTypeEnum(const SPIRVExtInst *DebugInst); DINode *transTypeTemplateParameter(const SPIRVExtInst *DebugInst); DINode *transTypeTemplateTemplateParameter(const SPIRVExtInst *DebugInst); DINode *transTypeTemplateParameterPack(const SPIRVExtInst *DebugInst); MDNode *transTypeTemplate(const SPIRVExtInst *DebugInst); DINode *transTypeFunction(const SPIRVExtInst *DebugInst); DINode *transTypePtrToMember(const SPIRVExtInst *DebugInst); DINode *transLexicalBlock(const SPIRVExtInst *DebugInst); DINode *transLexicalBlockDiscriminator(const SPIRVExtInst *DebugInst); DINode *transFunction(const SPIRVExtInst *DebugInst, bool IsMainSubprogram = false); DINode *transFunctionDefinition(const SPIRVExtInst *DebugInst); void transFunctionBody(DISubprogram *DIS, SPIRVId FuncId); DINode *transFunctionDecl(const SPIRVExtInst *DebugInst); MDNode *transEntryPoint(const SPIRVExtInst *DebugInst); MDNode *transGlobalVariable(const SPIRVExtInst *DebugInst); DINode *transLocalVariable(const SPIRVExtInst *DebugInst); DINode *transTypedef(const SPIRVExtInst *DebugInst); DINode *transTypeInheritance(const SPIRVExtInst *DebugInst, DIType *ChildClass = nullptr); DINode *transImportedEntry(const SPIRVExtInst *DebugInst); DINode *transModule(const SPIRVExtInst *DebugInst); MDNode *transExpression(const SPIRVExtInst *DebugInst); SPIRVModule *BM; Module *M; std::unordered_map> BuilderMap; SPIRVToLLVM *SPIRVReader; DICompileUnit *CU; bool Enable; std::unordered_map FileMap; std::unordered_map FuncMap; std::unordered_map DebugInstCache; struct SplitFileName { SplitFileName(const std::string &FileName); std::string BaseName; std::string Path; }; DIScope *getScope(const SPIRVEntry *ScopeInst); SPIRVExtInst *getDbgInst(const SPIRVId Id); DIBuilder &getDIBuilder(const SPIRVExtInst *DebugInst); template SPIRVExtInst *getDbgInst(const SPIRVId Id) { if (SPIRVExtInst *DI = getDbgInst(Id)) { if (DI->getExtOp() == OpCode) { return DI; } } return nullptr; } const std::string &getString(const SPIRVId Id); Optional getStringSourceContinued(const SPIRVId Id, SPIRVExtInst *DebugInst); SPIRVWord getConstantValueOrLiteral(const std::vector &, const SPIRVWord, const SPIRVExtInstSetKind); std::string findModuleProducer(); Optional> ParseChecksum(StringRef Text); // BuildIdentifier and StoragePath must both be set or both unset. // If StoragePath is empty both variables are unset and not valid. uint64_t BuildIdentifier{0}; std::string StoragePath{}; void setBuildIdentifierAndStoragePath(); }; } // namespace SPIRV #endif // SPIRVTOLLVMDBGTRAN_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToOCL.cpp000066400000000000000000001444621506646577400214470ustar00rootroot00000000000000//===- SPIRVToOCL.cpp - Transform SPIR-V builtins to OCL builtins------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements common transform of SPIR-V builtins to OCL builtins. // // Some of the visit functions are translations to OCL2.0 builtins, but they // are currently used also for OCL1.2, so theirs implementations are placed // in this pass as a common functionality for both versions. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spvtocl" #include "SPIRVToOCL.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/CommandLine.h" namespace SPIRV { void SPIRVToOCLBase::visitCallInst(CallInst &CI) { LLVM_DEBUG(dbgs() << "[visistCallInst] " << CI << '\n'); auto F = CI.getCalledFunction(); if (!F) return; OCLExtOpKind ExtOp; if (isSPIRVOCLExtInst(&CI, &ExtOp)) { switch (ExtOp) { case OpenCLLIB::Vloadn: case OpenCLLIB::Vloada_halfn: case OpenCLLIB::Vload_halfn: visitCallSPIRVVLoadn(&CI, ExtOp); break; case OpenCLLIB::Vstoren: case OpenCLLIB::Vstore_halfn: case OpenCLLIB::Vstorea_halfn: case OpenCLLIB::Vstore_half_r: case OpenCLLIB::Vstore_halfn_r: case OpenCLLIB::Vstorea_halfn_r: visitCallSPIRVVStore(&CI, ExtOp); break; case OpenCLLIB::Printf: { // TODO: Lower the printf instruction with the non-constant address space // format string to suitable for OpenCL representation if (dyn_cast(CI.getOperand(0)->getType()) ->getAddressSpace() == SPIR::TypeAttributeEnum::ATTR_CONST) visitCallSPIRVPrintf(&CI, ExtOp); break; } default: visitCallSPIRVOCLExt(&CI, ExtOp); break; } return; } auto MangledName = F->getName(); StringRef DemangledName; Op OC = OpNop; SPIRVBuiltinVariableKind BuiltinKind = SPIRVBuiltinVariableKind::BuiltInMax; if (!oclIsBuiltin(MangledName, DemangledName) || ((OC = getSPIRVFuncOC(DemangledName)) == OpNop && !getSPIRVBuiltin(DemangledName.str(), BuiltinKind))) return; LLVM_DEBUG(dbgs() << "DemangledName = " << DemangledName.str() << '\n' << "OpCode = " << OC << '\n' << "BuiltinKind = " << BuiltinKind << '\n'); if (BuiltinKind != SPIRVBuiltinVariableKind::BuiltInMax) { if (static_cast(BuiltinKind) >= internal::BuiltInSubDeviceIDINTEL && static_cast(BuiltinKind) <= internal::BuiltInGlobalHWThreadIDINTEL) return; visitCallSPIRVBuiltin(&CI, BuiltinKind); return; } if (OC == OpImageQuerySize || OC == OpImageQuerySizeLod) { visitCallSPRIVImageQuerySize(&CI); return; } if (OC == OpMemoryBarrier) { visitCallSPIRVMemoryBarrier(&CI); return; } if (OC == OpControlBarrier) { visitCallSPIRVControlBarrier(&CI); } if (isSplitBarrierINTELOpCode(OC)) { visitCallSPIRVSplitBarrierINTEL(&CI, OC); return; } if (isAtomicOpCode(OC)) { visitCallSPIRVAtomicBuiltin(&CI, OC); return; } if (isGroupOpCode(OC) || isGroupNonUniformOpcode(OC)) { visitCallSPIRVGroupBuiltin(&CI, OC); return; } if (isPipeOpCode(OC)) { visitCallSPIRVPipeBuiltin(&CI, OC); return; } if (isMediaBlockINTELOpcode(OC)) { visitCallSPIRVImageMediaBlockBuiltin(&CI, OC); return; } if (isIntelSubgroupOpCode(OC)) { visitCallSPIRVSubgroupINTELBuiltIn(&CI, OC); return; } if (isSubgroupAvcINTELEvaluateOpcode(OC)) { visitCallSPIRVAvcINTELEvaluateBuiltIn(&CI, OC); return; } if (isSubgroupAvcINTELInstructionOpCode(OC)) { visitCallSPIRVAvcINTELInstructionBuiltin(&CI, OC); return; } if (OC == OpBuildNDRange) { visitCallBuildNDRangeBuiltIn(&CI, OC, DemangledName); return; } if (OC == OpGenericCastToPtrExplicit) { visitCallGenericCastToPtrExplicitBuiltIn(&CI, OC); return; } if (isCvtOpCode(OC)) { visitCallSPIRVCvtBuiltin(&CI, OC, DemangledName); return; } if (OC == OpGroupAsyncCopy) { visitCallAsyncWorkGroupCopy(&CI, OC); return; } if (OC == OpGroupWaitEvents) { visitCallGroupWaitEvents(&CI, OC); return; } if (OC == OpImageSampleExplicitLod) { visitCallSPIRVImageSampleExplicitLodBuiltIn(&CI, OC); return; } if (OC == OpImageWrite) { visitCallSPIRVImageWriteBuiltIn(&CI, OC); return; } if (OC == OpImageRead) { visitCallSPIRVImageReadBuiltIn(&CI, OC); return; } if (OC == OpImageQueryOrder || OC == OpImageQueryFormat) { visitCallSPIRVImageQueryBuiltIn(&CI, OC); return; } if (OC == OpEnqueueKernel) { visitCallSPIRVEnqueueKernel(&CI, OC); return; } if (OC == OpGenericPtrMemSemantics) { visitCallSPIRVGenericPtrMemSemantics(&CI); return; } // Check if OC is OpenCL relational builtin except bitselect and select. auto IsOclRelationalOp = [](Op OC) { return isUnaryPredicateOpCode(OC) || OC == OpOrdered || OC == OpUnordered || OC == OpFOrdEqual || OC == OpFUnordNotEqual || OC == OpFOrdGreaterThan || OC == OpFOrdGreaterThanEqual || OC == OpFOrdLessThan || OC == OpFOrdLessThanEqual || OC == OpFOrdNotEqual; }; if (IsOclRelationalOp(OC)) { if (OC == OpAny || OC == OpAll) visitCallSPIRVAnyAll(&CI, OC); else visitCallSPIRVRelational(&CI, OC); return; } if (OC == OpReadClockKHR) { visitCallSPIRVReadClockKHR(&CI); return; } if (OC == internal::OpConvertFToBF16INTEL || OC == internal::OpConvertBF16ToFINTEL) { visitCallSPIRVBFloat16Conversions(&CI, OC); return; } if (OCLSPIRVBuiltinMap::rfind(OC)) visitCallSPIRVBuiltin(&CI, OC); } void SPIRVToOCLBase::visitCastInst(CastInst &Cast) { if (!isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast) && !isa(Cast)) return; Type const *SrcTy = Cast.getSrcTy(); Type *DstVecTy = Cast.getDestTy(); // Leave scalar casts as is. Skip boolean vector casts becase there // are no suitable OCL built-ins. if (!DstVecTy->isVectorTy() || SrcTy->getScalarSizeInBits() == 1 || DstVecTy->getScalarSizeInBits() == 1) return; // Assemble built-in name -> convert_gentypeN std::string CastBuiltInName(kOCLBuiltinName::ConvertPrefix); // Check if this is 'floating point -> unsigned integer' cast CastBuiltInName += mapLLVMTypeToOCLType(DstVecTy, !isa(Cast)); // Replace LLVM conversion instruction with call to conversion built-in BuiltinFuncMangleInfo Mangle; // It does matter if the source is unsigned integer or not. SExt is for // signed source, ZExt and UIToFPInst are for unsigned source. if (isa(Cast) || isa(Cast)) Mangle.addUnsignedArg(0); AttributeList Attributes; CallInst *Call = addCallInst(M, CastBuiltInName, DstVecTy, Cast.getOperand(0), &Attributes, &Cast, &Mangle, Cast.getName(), false); Cast.replaceAllUsesWith(Call); Cast.eraseFromParent(); } void SPIRVToOCLBase::visitCallSPRIVImageQuerySize(CallInst *CI) { // Get image type SmallVector ParamTys; getParameterTypes(CI, ParamTys); StructType *ImgTy = ParamTys[0]; assert(ImgTy && ImgTy->isOpaque() && "image type must be an opaque structure"); StringRef ImgTyName = ImgTy->getName(); assert(ImgTyName.startswith("opencl.image") && "not an OCL image type"); unsigned ImgDim = 0; bool ImgArray = false; if (ImgTyName.startswith("opencl.image1d")) { ImgDim = 1; } else if (ImgTyName.startswith("opencl.image2d")) { ImgDim = 2; } else if (ImgTyName.startswith("opencl.image3d")) { ImgDim = 3; } assert(ImgDim != 0 && "unexpected image dimensionality"); if (ImgTyName.count("_array_") != 0) { ImgArray = true; } AttributeList Attributes = CI->getCalledFunction()->getAttributes(); BuiltinFuncMangleInfo Mangle; Mangle.getTypeMangleInfo(0).PointerElementType.setPointer(ImgTy); Type *Int32Ty = Type::getInt32Ty(*Ctx); Instruction *GetImageSize = nullptr; if (ImgDim == 1) { // OpImageQuerySize from non-arrayed 1d image is always translated // into get_image_width returning scalar argument GetImageSize = addCallInst(M, kOCLBuiltinName::GetImageWidth, Int32Ty, CI->getArgOperand(0), &Attributes, CI, &Mangle, CI->getName(), false); // The width of integer type returning by OpImageQuerySize[Lod] may // differ from i32 if (CI->getType()->getScalarType() != Int32Ty) { GetImageSize = CastInst::CreateIntegerCast(GetImageSize, CI->getType()->getScalarType(), false, CI->getName(), CI); } } else { assert((ImgDim == 2 || ImgDim == 3) && "invalid image type"); assert(CI->getType()->isVectorTy() && "this code can handle vector result type only"); // get_image_dim returns int2 and int4 for 2d and 3d images respecitvely. const unsigned ImgDimRetEls = ImgDim == 2 ? 2 : 4; VectorType *RetTy = FixedVectorType::get(Int32Ty, ImgDimRetEls); GetImageSize = addCallInst(M, kOCLBuiltinName::GetImageDim, RetTy, CI->getArgOperand(0), &Attributes, CI, &Mangle, CI->getName(), false); // The width of integer type returning by OpImageQuerySize[Lod] may // differ from i32 if (CI->getType()->getScalarType() != Int32Ty) { GetImageSize = CastInst::CreateIntegerCast( GetImageSize, FixedVectorType::get( CI->getType()->getScalarType(), cast(GetImageSize->getType())->getNumElements()), false, CI->getName(), CI); } } if (ImgArray || ImgDim == 3) { auto *VecTy = cast(CI->getType()); const unsigned ImgQuerySizeRetEls = VecTy->getNumElements(); if (ImgDim == 1) { // get_image_width returns scalar result while OpImageQuerySize // for image1d_array_t returns <2 x i32> vector. assert(ImgQuerySizeRetEls == 2 && "OpImageQuerySize[Lod] must return <2 x iN> vector type"); GetImageSize = InsertElementInst::Create( UndefValue::get(VecTy), GetImageSize, ConstantInt::get(Int32Ty, 0), CI->getName(), CI); } else { // get_image_dim and OpImageQuerySize returns different vector // types for arrayed and 3d images. SmallVector MaskEls; for (unsigned Idx = 0; Idx < ImgQuerySizeRetEls; ++Idx) MaskEls.push_back(ConstantInt::get(Int32Ty, Idx)); Constant *Mask = ConstantVector::get(MaskEls); GetImageSize = new ShuffleVectorInst( GetImageSize, UndefValue::get(GetImageSize->getType()), Mask, CI->getName(), CI); } } if (ImgArray) { assert((ImgDim == 1 || ImgDim == 2) && "invalid image array type"); // Insert get_image_array_size to the last position of the resulting vector. auto *VecTy = cast(CI->getType()); Type *SizeTy = Type::getIntNTy(*Ctx, M->getDataLayout().getPointerSizeInBits(0)); Instruction *GetImageArraySize = addCallInst( M, kOCLBuiltinName::GetImageArraySize, SizeTy, CI->getArgOperand(0), &Attributes, CI, &Mangle, CI->getName(), false); // The width of integer type returning by OpImageQuerySize[Lod] may // differ from size_t which is returned by get_image_array_size if (GetImageArraySize->getType() != VecTy->getElementType()) { GetImageArraySize = CastInst::CreateIntegerCast( GetImageArraySize, VecTy->getElementType(), false, CI->getName(), CI); } GetImageSize = InsertElementInst::Create( GetImageSize, GetImageArraySize, ConstantInt::get(Int32Ty, VecTy->getNumElements() - 1), CI->getName(), CI); } assert(GetImageSize && "must not be null"); CI->replaceAllUsesWith(GetImageSize); CI->eraseFromParent(); } std::string SPIRVToOCLBase::getUniformArithmeticBuiltinName(CallInst *CI, Op OC) { assert(isUniformArithmeticOpCode(OC) && "Not intended to handle other than uniform arithmetic opcodes!"); auto FuncName = OCLSPIRVBuiltinMap::rmap(OC); std::string Prefix = getGroupBuiltinPrefix(CI); std::string Op = FuncName; Op.erase(0, strlen(kSPIRVName::GroupPrefix)); // unsigned prefix cannot be removed yet, as it is necessary to properly // mangle the function bool Unsigned = Op.front() == 'u'; if (!Unsigned) Op = Op.erase(0, 1); std::string GroupOp; auto GO = getArgAs(CI, 1); switch (GO) { case GroupOperationReduce: GroupOp = "reduce"; break; case GroupOperationInclusiveScan: GroupOp = "scan_inclusive"; break; case GroupOperationExclusiveScan: GroupOp = "scan_exclusive"; break; default: llvm_unreachable("Unsupported group operation!"); break; } return Prefix + kSPIRVName::GroupPrefix + GroupOp + "_" + Op; } std::string SPIRVToOCLBase::getNonUniformArithmeticBuiltinName(CallInst *CI, Op OC) { assert(isNonUniformArithmeticOpCode(OC) && "Not intended to handle other than non uniform arithmetic opcodes!"); std::string Prefix = getGroupBuiltinPrefix(CI); assert((Prefix == kOCLBuiltinName::SubPrefix) && "Workgroup scope is not supported for OpGroupNonUniform opcodes"); auto FuncName = OCLSPIRVBuiltinMap::rmap(OC); std::string Op = FuncName; Op.erase(0, strlen(kSPIRVName::GroupNonUniformPrefix)); if (!isGroupLogicalOpCode(OC)) { // unsigned prefix cannot be removed yet, as it is necessary to properly // mangle the function const char Sign = Op.front(); bool Signed = (Sign == 'i' || Sign == 'f' || Sign == 's'); if (Signed) Op = Op.erase(0, 1); else assert((Sign == 'u') && "Incorrect sign!"); } else { // LogicalOpcode assert( (Op == "logical_iand" || Op == "logical_ior" || Op == "logical_ixor") && "Incorrect logical operation"); Op = Op.erase(8, 1); } std::string GroupOp; std::string GroupPrefix = kSPIRVName::GroupNonUniformPrefix; auto GO = getArgAs(CI, 1); switch (GO) { case GroupOperationReduce: GroupOp = "reduce"; break; case GroupOperationInclusiveScan: GroupOp = "scan_inclusive"; break; case GroupOperationExclusiveScan: GroupOp = "scan_exclusive"; break; case GroupOperationClusteredReduce: GroupOp = "clustered_reduce"; // OpenCL clustered builtin has no non_uniform prefix, ex. // sub_group_reduce_clustered_logical_and GroupPrefix = kSPIRVName::GroupPrefix; break; default: llvm_unreachable("Unsupported group operation!"); break; } return Prefix + GroupPrefix + GroupOp + "_" + Op; } std::string SPIRVToOCLBase::getBallotBuiltinName(CallInst *CI, Op OC) { assert((OC == OpGroupNonUniformBallotBitCount) && "Not inteded to handle other opcodes than " "OpGroupNonUniformBallotBitCount!"); std::string Prefix = getGroupBuiltinPrefix(CI); assert( (Prefix == kOCLBuiltinName::SubPrefix) && "Workgroup scope is not supported for OpGroupNonUniformBallotBitCount"); std::string GroupOp; auto GO = getArgAs(CI, 1); switch (GO) { case GroupOperationReduce: GroupOp = "bit_count"; break; case GroupOperationInclusiveScan: GroupOp = "inclusive_scan"; break; case GroupOperationExclusiveScan: GroupOp = "exclusive_scan"; break; default: llvm_unreachable("Unsupported group operation!"); break; } return Prefix + kSPIRVName::GroupPrefix + "ballot_" + GroupOp; } std::string SPIRVToOCLBase::getRotateBuiltinName(CallInst *CI, Op OC) { assert((OC == OpGroupNonUniformRotateKHR) && "Not intended to handle other opcodes"); std::string Prefix = getGroupBuiltinPrefix(CI); assert((Prefix == kOCLBuiltinName::SubPrefix) && "Workgroup scope is not supported for OpGroupNonUniformRotateKHR"); std::string OptionalClustered; if (CI->arg_size() == 4) OptionalClustered = "clustered_"; return Prefix + kSPIRVName::GroupPrefix + OptionalClustered + "rotate"; } std::string SPIRVToOCLBase::groupOCToOCLBuiltinName(CallInst *CI, Op OC) { if (OC == OpGroupNonUniformRotateKHR) return getRotateBuiltinName(CI, OC); auto FuncName = OCLSPIRVBuiltinMap::rmap(OC); assert(FuncName.find(kSPIRVName::GroupPrefix) == 0); if (!hasGroupOperation(OC)) { /// Transform OpenCL group builtin function names from group_ /// to work_group_ and sub_group_. FuncName = getGroupBuiltinPrefix(CI) + FuncName; } else { // Opcodes with group operation parameter if (isUniformArithmeticOpCode(OC)) FuncName = getUniformArithmeticBuiltinName(CI, OC); else if (isNonUniformArithmeticOpCode(OC)) FuncName = getNonUniformArithmeticBuiltinName(CI, OC); else if (OC == OpGroupNonUniformBallotBitCount) FuncName = getBallotBuiltinName(CI, OC); else llvm_unreachable("Unsupported opcode!"); } return FuncName; } /// Return true if the original boolean return type needs to be changed to i32 /// when mapping the SPIR-V op to an OpenCL builtin. static bool needsInt32RetTy(Op OC) { return OC == OpGroupAny || OC == OpGroupAll || OC == OpGroupNonUniformAny || OC == OpGroupNonUniformAll || OC == OpGroupNonUniformAllEqual || OC == OpGroupNonUniformElect || OC == OpGroupNonUniformInverseBallot || OC == OpGroupNonUniformBallotBitExtract || isGroupLogicalOpCode(OC); } void SPIRVToOCLBase::visitCallSPIRVGroupBuiltin(CallInst *CI, Op OC) { auto FuncName = groupOCToOCLBuiltinName(CI, OC); auto ModifyArguments = [=](CallInst *, std::vector &Args, llvm::Type *&RetTy) { Type *Int32Ty = Type::getInt32Ty(*Ctx); bool HasArg0ExtendedToi32 = OC == OpGroupAny || OC == OpGroupAll || OC == OpGroupNonUniformAny || OC == OpGroupNonUniformAll || OC == OpGroupNonUniformBallot || isGroupLogicalOpCode(OC); /// Remove Group Operation argument, /// as in OpenCL representation this is included in the function name Args.erase(Args.begin(), Args.begin() + (hasGroupOperation(OC) ? 2 : 1)); // Handle function arguments if (OC == OpGroupBroadcast) expandVector(CI, Args, 1); else if (HasArg0ExtendedToi32) Args[0] = CastInst::CreateZExtOrBitCast(Args[0], Int32Ty, "", CI); // Handle function return type if (needsInt32RetTy(OC)) RetTy = Int32Ty; return FuncName; }; auto ModifyRetTy = [=](CallInst *CI) -> Instruction * { if (needsInt32RetTy(OC)) { // The OpenCL builtin returns a non-zero integer value. Convert to a // boolean value. Constant *Zero = ConstantInt::get(CI->getType(), 0); return new ICmpInst(CI->getNextNode(), CmpInst::ICMP_NE, CI, Zero); } else return CI; }; assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); SmallVector ArgAttrs; for (int I = (hasGroupOperation(OC) ? 2 : 1); I < (int)Attrs.getNumAttrSets() - 2; I++) ArgAttrs.push_back(Attrs.getParamAttrs(I)); Attrs = AttributeList::get(*Ctx, Attrs.getFnAttrs(), Attrs.getRetAttrs(), ArgAttrs); mutateCallInstOCL(M, CI, ModifyArguments, ModifyRetTy, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVPipeBuiltin(CallInst *CI, Op OC) { auto DemangledName = OCLSPIRVBuiltinMap::rmap(OC); bool HasScope = DemangledName.find(kSPIRVName::GroupPrefix) == 0; if (HasScope) DemangledName = getGroupBuiltinPrefix(CI) + DemangledName; assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { if (HasScope) Args.erase(Args.begin(), Args.begin() + 1); if (!(OC == OpReadPipe || OC == OpWritePipe || OC == OpReservedReadPipe || OC == OpReservedWritePipe || OC == OpReadPipeBlockingINTEL || OC == OpWritePipeBlockingINTEL)) return DemangledName; auto &P = Args[Args.size() - 3]; auto T = P->getType(); assert(isa(T)); auto *NewTy = PointerType::getInt8PtrTy(*Ctx, SPIRAS_Generic); if (T != NewTy) { P = CastInst::CreatePointerBitCastOrAddrSpaceCast(P, NewTy, "", CI); } return DemangledName; }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVImageMediaBlockBuiltin(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // Moving the first argument to the end. std::rotate(Args.rbegin(), Args.rend() - 1, Args.rend()); Type *RetType = CI->getType(); if (OC == OpSubgroupImageMediaBlockWriteINTEL) { assert(Args.size() >= 4 && "Wrong media block write signature"); RetType = Args.at(3)->getType(); // texel type } unsigned int BitWidth = RetType->getScalarSizeInBits(); std::string FuncPostfix; if (BitWidth == 8) FuncPostfix = "_uc"; else if (BitWidth == 16) FuncPostfix = "_us"; else if (BitWidth == 32) FuncPostfix = "_ui"; else assert(0 && "Unsupported texel type!"); if (auto *VecTy = dyn_cast(RetType)) { unsigned int NumEl = VecTy->getNumElements(); assert((NumEl == 2 || NumEl == 4 || NumEl == 8 || NumEl == 16) && "Wrong function type!"); FuncPostfix += std::to_string(NumEl); } return OCLSPIRVBuiltinMap::rmap(OC) + FuncPostfix; }, &Attrs); } void SPIRVToOCLBase::visitCallBuildNDRangeBuiltIn(CallInst *CI, Op OC, StringRef DemangledName) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *Call, std::vector &Args) { assert(Args.size() == 3); // OpenCL built-in has another order of parameters. auto *GlobalWorkSize = Args[0]; auto *LocalWorkSize = Args[1]; auto *GlobalWorkOffset = Args[2]; Args[0] = GlobalWorkOffset; Args[1] = GlobalWorkSize; Args[2] = LocalWorkSize; // __spirv_BuildNDRange_nD, drop __spirv_ StringRef S = DemangledName; S = S.drop_front(strlen(kSPIRVName::Prefix)); SmallVector Split; // BuildNDRange_nD S.split(Split, kSPIRVPostfix::Divider, /*MaxSplit=*/-1, /*KeepEmpty=*/false); assert(Split.size() >= 2 && "Invalid SPIRV function name"); // Cut _nD and add it to function name. return std::string(kOCLBuiltinName::NDRangePrefix) + Split[1].substr(0, 3).str(); }, &Attrs); } void SPIRVToOCLBase::visitCallGenericCastToPtrExplicitBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *Call, std::vector &Args) { auto AddrSpace = static_cast( CI->getType()->getPointerAddressSpace()); // The instruction has two arguments, whereas ocl built-in has only one // argument. Args.pop_back(); switch (AddrSpace) { case SPIRAS_Global: return std::string(kOCLBuiltinName::ToGlobal); case SPIRAS_Local: return std::string(kOCLBuiltinName::ToLocal); case SPIRAS_Private: return std::string(kOCLBuiltinName::ToPrivate); default: llvm_unreachable("Invalid address space"); return std::string(); } }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVCvtBuiltin(CallInst *CI, Op OC, StringRef DemangledName) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *Call, std::vector &Args) { std::string CastBuiltInName; if (isCvtFromUnsignedOpCode(OC)) CastBuiltInName = "u"; CastBuiltInName += kOCLBuiltinName::ConvertPrefix; Type *DstTy = Call->getType(); CastBuiltInName += mapLLVMTypeToOCLType(DstTy, !isCvtToUnsignedOpCode(OC)); if (DemangledName.find("_sat") != StringRef::npos || isSatCvtOpCode(OC)) CastBuiltInName += "_sat"; Value *Src = Call->getOperand(0); assert(Src && "Invalid SPIRV convert builtin call"); Type *SrcTy = Src->getType(); auto Loc = DemangledName.find("_rt"); if (Loc != StringRef::npos && !(isa(SrcTy) && isa(DstTy))) CastBuiltInName += DemangledName.substr(Loc, 4).str(); return CastBuiltInName; }, &Attrs); } void SPIRVToOCLBase::visitCallAsyncWorkGroupCopy(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // First argument of AsyncWorkGroupCopy instruction is Scope, OCL // built-in async_work_group_strided_copy doesn't have this argument Args.erase(Args.begin()); return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } void SPIRVToOCLBase::visitCallGroupWaitEvents(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // First argument of GroupWaitEvents instruction is Scope, OCL // built-in wait_group_events doesn't have this argument Args.erase(Args.begin()); return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } static std::string getTypeSuffix(Type *T, bool IsSigned) { std::string Suffix; Type *ST = T->getScalarType(); if (ST->isHalfTy()) Suffix = "h"; else if (ST->isFloatTy()) Suffix = "f"; else if (IsSigned) Suffix = "i"; else Suffix = "ui"; return Suffix; } void SPIRVToOCLBase::mutateArgsForImageOperands(std::vector &Args, unsigned ImOpArgIndex, bool &IsSigned) { // Default to signed. IsSigned = true; if (Args.size() > ImOpArgIndex) { ConstantInt *ImOp = dyn_cast(Args[ImOpArgIndex]); uint64_t ImOpValue = 0; if (ImOp) ImOpValue = ImOp->getZExtValue(); unsigned SignZeroExtMasks = ImageOperandsMask::ImageOperandsSignExtendMask | ImageOperandsMask::ImageOperandsZeroExtendMask; // If one of the SPIR-V 1.4 SignExtend/ZeroExtend operands is present, take // it into account and drop the mask. if (ImOpValue & SignZeroExtMasks) { if (ImOpValue & ImageOperandsMask::ImageOperandsZeroExtendMask) IsSigned = false; ImOpValue &= ~SignZeroExtMasks; Args[ImOpArgIndex] = getInt32(M, ImOpValue); ImOp = cast(Args[ImOpArgIndex]); } // Drop "Image Operands" argument. Args.erase(Args.begin() + ImOpArgIndex); if (Args.size() > ImOpArgIndex) { ConstantFP *LodVal = dyn_cast(Args[ImOpArgIndex]); // If the image operand is LOD and its value is zero, drop it too. if (LodVal && LodVal->isNullValue() && ImOpValue == ImageOperandsMask::ImageOperandsLodMask) Args.erase(Args.begin() + ImOpArgIndex, Args.end()); } } } void SPIRVToOCLBase::visitCallSPIRVImageSampleExplicitLodBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); CallInst *CallSampledImg = cast(CI->getArgOperand(0)); SmallVector ParamTys; getParameterTypes(CallSampledImg, ParamTys); StringRef ImageTypeName; bool IsDepthImage = false; if (isOCLImageStructType(ParamTys[0], &ImageTypeName)) IsDepthImage = ImageTypeName.contains("_depth_"); auto ModifyArguments = [=](CallInst *, std::vector &Args, llvm::Type *&RetTy) { auto Img = CallSampledImg->getArgOperand(0); if (!Img->getType()->isOpaquePointerTy()) assert(isOCLImageStructType( Img->getType()->getNonOpaquePointerElementType())); auto Sampler = CallSampledImg->getArgOperand(1); Args[0] = Img; Args.insert(Args.begin() + 1, Sampler); bool IsSigned; mutateArgsForImageOperands(Args, 3, IsSigned); if (CallSampledImg->hasOneUse()) { CallSampledImg->replaceAllUsesWith( UndefValue::get(CallSampledImg->getType())); CallSampledImg->dropAllReferences(); CallSampledImg->eraseFromParent(); } Type *T = CI->getType(); if (auto VT = dyn_cast(T)) T = VT->getElementType(); RetTy = IsDepthImage ? T : CI->getType(); return std::string(kOCLBuiltinName::SampledReadImage) + getTypeSuffix(T, IsSigned); }; auto ModifyRetTy = [=](CallInst *NewCI) -> Instruction * { if (IsDepthImage) { auto Ins = InsertElementInst::Create( UndefValue::get(FixedVectorType::get(NewCI->getType(), 4)), NewCI, getSizet(M, 0)); Ins->insertAfter(NewCI); return Ins; } return NewCI; }; mutateCallInstOCL(M, CI, ModifyArguments, ModifyRetTy, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVImageWriteBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { llvm::Type *T = Args[2]->getType(); bool IsSigned; mutateArgsForImageOperands(Args, 3, IsSigned); if (Args.size() > 3) { std::swap(Args[2], Args[3]); } return std::string(kOCLBuiltinName::WriteImage) + getTypeSuffix(T, IsSigned); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVImageReadBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { bool IsSigned; mutateArgsForImageOperands(Args, 2, IsSigned); llvm::Type *T = CI->getType(); return std::string(kOCLBuiltinName::ReadImage) + getTypeSuffix(T, IsSigned); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVImageQueryBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); CI = mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); unsigned int Offset = 0; if (OC == OpImageQueryFormat) Offset = OCLImageChannelDataTypeOffset; else if (OC == OpImageQueryOrder) Offset = OCLImageChannelOrderOffset; else llvm_unreachable("Unsupported opcode"); auto *Sub = BinaryOperator::CreateSub(CI, getInt32(M, Offset), "", CI->getNextNode()); for (auto &Use : CI->uses()) { if (Use.getUser() == Sub) continue; Use.set(Sub); } } void SPIRVToOCLBase::visitCallSPIRVSubgroupINTELBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { std::stringstream Name; Type *DataTy = nullptr; switch (OC) { case OpSubgroupBlockReadINTEL: case OpSubgroupImageBlockReadINTEL: Name << "intel_sub_group_block_read"; DataTy = CI->getType(); break; case OpSubgroupBlockWriteINTEL: Name << "intel_sub_group_block_write"; DataTy = CI->getOperand(1)->getType(); break; case OpSubgroupImageBlockWriteINTEL: Name << "intel_sub_group_block_write"; DataTy = CI->getOperand(2)->getType(); break; default: return OCLSPIRVBuiltinMap::rmap(OC); } assert(DataTy && "Intel subgroup block builtins should have data type"); unsigned VectorNumElements = 1; if (FixedVectorType *VT = dyn_cast(DataTy)) VectorNumElements = VT->getNumElements(); unsigned ElementBitSize = DataTy->getScalarSizeInBits(); Name << getIntelSubgroupBlockDataPostfix(ElementBitSize, VectorNumElements); return Name.str(); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVAvcINTELEvaluateBuiltIn(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // There are three types of AVC Intel Evaluate opcodes: // 1. With multi reference images - does not use OpVmeImageINTEL opcode // for reference images // 2. With dual reference images - uses two OpVmeImageINTEL opcodes for // reference image // 3. With single reference image - uses one OpVmeImageINTEL opcode for // reference image StringRef FnName = CI->getCalledFunction()->getName(); int NumImages = 0; if (FnName.contains("SingleReference")) NumImages = 2; else if (FnName.contains("DualReference")) NumImages = 3; else if (FnName.contains("MultiReference")) NumImages = 1; else if (FnName.contains("EvaluateIpe")) NumImages = 1; auto EraseVmeImageCall = [](CallInst *CI) { if (CI->hasOneUse()) { CI->replaceAllUsesWith(UndefValue::get(CI->getType())); CI->dropAllReferences(); CI->eraseFromParent(); } }; if (NumImages) { CallInst *SrcImage = cast(Args[0]); if (NumImages == 1) { // Multi reference opcode - remove src image OpVmeImageINTEL opcode // and replace it with corresponding OpImage and OpSampler arguments size_t SamplerPos = Args.size() - 1; Args.erase(Args.begin(), Args.begin() + 1); Args.insert(Args.begin(), SrcImage->getOperand(0)); Args.insert(Args.begin() + SamplerPos, SrcImage->getOperand(1)); EraseVmeImageCall(SrcImage); } else { CallInst *FwdRefImage = cast(Args[1]); CallInst *BwdRefImage = NumImages == 3 ? cast(Args[2]) : nullptr; // Single reference opcode - remove src and ref image // OpVmeImageINTEL opcodes and replace them with src and ref OpImage // opcodes and OpSampler Args.erase(Args.begin(), Args.begin() + NumImages); // insert source OpImage and OpSampler auto SrcOps = SrcImage->args(); Args.insert(Args.begin(), SrcOps.begin(), SrcOps.end()); // insert reference OpImage Args.insert(Args.begin() + 1, FwdRefImage->getOperand(0)); EraseVmeImageCall(SrcImage); EraseVmeImageCall(FwdRefImage); if (BwdRefImage) { // Dual reference opcode - insert second reference OpImage // argument Args.insert(Args.begin() + 2, BwdRefImage->getOperand(0)); EraseVmeImageCall(BwdRefImage); } } } else llvm_unreachable("invalid avc instruction"); return OCLSPIRVSubgroupAVCIntelBuiltinMap::rmap(OC); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVGenericPtrMemSemantics(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { return OCLSPIRVBuiltinMap::rmap(OpGenericPtrMemSemantics); }, [=](CallInst *CI) -> Instruction * { auto *Shl = BinaryOperator::CreateShl(CI, getInt32(M, 8), ""); Shl->insertAfter(CI); return Shl; }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVBFloat16Conversions(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Type *ArgTy = CI->getOperand(0)->getType(); std::string N = ArgTy->isVectorTy() ? std::to_string(cast(ArgTy)->getNumElements()) : ""; std::string Name; switch (static_cast(OC)) { case internal::OpConvertFToBF16INTEL: Name = "intel_convert_bfloat16" + N + "_as_ushort" + N; break; case internal::OpConvertBF16ToFINTEL: Name = "intel_convert_as_bfloat16" + N + "_float" + N; break; default: break; // do nothing } return Name; }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVBuiltin(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVBuiltin(CallInst *CI, SPIRVBuiltinVariableKind Kind) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return SPIRSPIRVBuiltinVariableMap::rmap(Kind); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVAvcINTELInstructionBuiltin(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCLSPIRVSubgroupAVCIntelBuiltinMap::rmap(OC); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVOCLExt(CallInst *CI, OCLExtOpKind Kind) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCLExtOpMap::map(Kind); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVVLoadn(CallInst *CI, OCLExtOpKind Kind) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { std::string Name = OCLExtOpMap::map(Kind); if (ConstantInt *C = dyn_cast(Args.back())) { uint64_t NumComponents = C->getZExtValue(); std::stringstream SS; SS << NumComponents; Name.replace(Name.find("n"), 1, SS.str()); } Args.pop_back(); return Name; }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVVStore(CallInst *CI, OCLExtOpKind Kind) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { std::string Name = OCLExtOpMap::map(Kind); if (Kind == OpenCLLIB::Vstore_half_r || Kind == OpenCLLIB::Vstore_halfn_r || Kind == OpenCLLIB::Vstorea_halfn_r) { auto C = cast(Args.back()); auto RoundingMode = static_cast(C->getZExtValue()); Name.replace(Name.find("_r"), 2, std::string("_") + SPIRSPIRVFPRoundingModeMap::rmap(RoundingMode)); Args.pop_back(); } if (Kind == OpenCLLIB::Vstore_halfn || Kind == OpenCLLIB::Vstore_halfn_r || Kind == OpenCLLIB::Vstorea_halfn || Kind == OpenCLLIB::Vstorea_halfn_r || Kind == OpenCLLIB::Vstoren) { if (auto DataType = dyn_cast(Args[0]->getType())) { uint64_t NumElements = DataType->getElementCount().getFixedValue(); assert((NumElements == 2 || NumElements == 3 || NumElements == 4 || NumElements == 8 || NumElements == 16) && "Unsupported vector size for vstore instruction!"); std::stringstream SS; SS << NumElements; Name.replace(Name.find("n"), 1, SS.str()); } } return Name; }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVPrintf(CallInst *CI, OCLExtOpKind Kind) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); CallInst *NewCI = mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCLExtOpMap::map(OpenCLLIB::Printf); }, &Attrs); // Clang represents printf function without mangling std::string TargetName = "printf"; if (Function *F = M->getFunction(TargetName)) NewCI->setCalledFunction(F); else NewCI->getCalledFunction()->setName(TargetName); } void SPIRVToOCLBase::visitCallSPIRVAnyAll(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { Type *Int8Ty = Type::getInt8Ty(*Ctx); auto *OldArg = CI->getOperand(0); auto *OldArgTy = cast(OldArg->getType()); if (Int8Ty != OldArgTy->getElementType()) { auto *NewArgTy = FixedVectorType::get(Int8Ty, OldArgTy->getNumElements()); auto *NewArg = CastInst::CreateSExtOrBitCast(OldArg, NewArgTy, "", CI); Args[0] = NewArg; } RetTy = Type::getInt32Ty(*Ctx); return OCLSPIRVBuiltinMap::rmap(OC); }, [=](CallInst *NewCI) -> Instruction * { return CastInst::CreateTruncOrBitCast(NewCI, CI->getType(), "", NewCI->getNextNode()); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVRelational(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector & /*Args*/, Type *&RetTy) { Type *IntTy = Type::getInt32Ty(*Ctx); RetTy = IntTy; if (CI->getType()->isVectorTy()) { auto *OpElemTy = cast(CI->getOperand(0)->getType()) ->getElementType(); if (OpElemTy->isDoubleTy()) IntTy = Type::getInt64Ty(*Ctx); if (OpElemTy->isHalfTy()) IntTy = Type::getInt16Ty(*Ctx); RetTy = FixedVectorType::get( IntTy, cast(CI->getType())->getNumElements()); } return OCLSPIRVBuiltinMap::rmap(OC); }, [=](CallInst *NewCI) -> Instruction * { return CastInst::CreateTruncOrBitCast(NewCI, CI->getType(), "", NewCI->getNextNode()); }, &Attrs); } void SPIRVToOCLBase::visitCallSPIRVReadClockKHR(CallInst *CI) { std::string Name("clock_read_"); if (CI->getType()->isVectorTy()) Name += "hilo_"; // Encode the scope (taken from the argument) in the function name. ConstantInt *ScopeOp = cast(CI->getArgOperand(0)); switch (static_cast(ScopeOp->getZExtValue())) { case ScopeDevice: Name += "device"; break; case ScopeWorkgroup: Name += "work_group"; break; case ScopeSubgroup: Name += "sub_group"; break; default: break; } mutateCallInstOCL(M, CI, [=](CallInst *Call, std::vector &Args) { Args.pop_back(); return Name; }); } std::string SPIRVToOCLBase::getGroupBuiltinPrefix(CallInst *CI) { std::string Prefix; auto ES = getArgAsScope(CI, 0); switch (ES) { case ScopeWorkgroup: Prefix = kOCLBuiltinName::WorkPrefix; break; case ScopeSubgroup: Prefix = kOCLBuiltinName::SubPrefix; break; default: llvm_unreachable("Invalid execution scope"); } return Prefix; } std::string SPIRVToOCLBase::getOCLImageOpaqueType(SmallVector &Postfixes) { SmallVector Ops; for (unsigned I = 1; I < 8; ++I) Ops.push_back(atoi(Postfixes[I].c_str())); SPIRVTypeImageDescriptor Desc(static_cast(Ops[0]), Ops[1], Ops[2], Ops[3], Ops[4], Ops[5]); std::string OCLStructName = std::string(kSPR2TypeName::OCLPrefix) + rmap(Desc); SPIRVAccessQualifierKind Acc = static_cast(Ops[6]); insertImageNameAccessQualifier(Acc, OCLStructName); return OCLStructName; } std::string SPIRVToOCLBase::getOCLPipeOpaqueType(SmallVector &Postfixes) { assert(Postfixes.size() == 1); unsigned PipeAccess = atoi(Postfixes[0].c_str()); assert((PipeAccess == AccessQualifierReadOnly || PipeAccess == AccessQualifierWriteOnly) && "Invalid access qualifier"); return PipeAccess ? kSPR2TypeName::PipeWO : kSPR2TypeName::PipeRO; } void SPIRVToOCLBase::translateOpaqueTypes() { for (auto *S : M->getIdentifiedStructTypes()) { StringRef STName = S->getStructName(); bool IsSPIRVOpaque = S->isOpaque() && STName.startswith(kSPIRVTypeName::PrefixAndDelim); if (!IsSPIRVOpaque) continue; S->setName(translateOpaqueType(STName)); } } std::string SPIRVToOCLBase::translateOpaqueType(StringRef STName) { if (!STName.startswith(kSPIRVTypeName::PrefixAndDelim)) return STName.str(); SmallVector Postfixes; std::string DecodedST = decodeSPIRVTypeName(STName, Postfixes); if (!SPIRVOpaqueTypeOpCodeMap::find(DecodedST)) return STName.str(); Op OP = SPIRVOpaqueTypeOpCodeMap::map(DecodedST); std::string OCLOpaqueName; if (OP == OpTypeImage) OCLOpaqueName = getOCLImageOpaqueType(Postfixes); else if (OP == OpTypePipe) OCLOpaqueName = getOCLPipeOpaqueType(Postfixes); else if (isSubgroupAvcINTELTypeOpCode(OP)) OCLOpaqueName = OCLSubgroupINTELTypeOpCodeMap::rmap(OP); else if (isOpaqueGenericTypeOpCode(OP)) OCLOpaqueName = OCLOpaqueTypeOpCodeMap::rmap(OP); else return STName.str(); return OCLOpaqueName; } void SPIRVToOCLBase::getParameterTypes(CallInst *CI, SmallVectorImpl &Tys) { ::getParameterTypes(CI, Tys); for (auto &Ty : Tys) { if (!Ty) continue; StringRef STName = Ty->getStructName(); bool IsSPIRVOpaque = Ty->isOpaque() && STName.startswith(kSPIRVTypeName::PrefixAndDelim); if (!IsSPIRVOpaque) continue; std::string NewName = translateOpaqueType(STName); if (NewName != STName) Ty = getOrCreateOpaqueStructType(M, NewName); }; } void addSPIRVBIsLoweringPass(ModulePassManager &PassMgr, SPIRV::BIsRepresentation BIsRep) { switch (BIsRep) { case SPIRV::BIsRepresentation::OpenCL12: PassMgr.addPass(SPIRVToOCL12Pass()); break; case SPIRV::BIsRepresentation::OpenCL20: PassMgr.addPass(SPIRVToOCL20Pass()); break; case SPIRV::BIsRepresentation::SPIRVFriendlyIR: // nothing to do, already done break; } } } // namespace SPIRV ModulePass * llvm::createSPIRVBIsLoweringPass(Module &M, SPIRV::BIsRepresentation BIsRepresentation) { switch (BIsRepresentation) { case SPIRV::BIsRepresentation::OpenCL12: return createSPIRVToOCL12Legacy(); case SPIRV::BIsRepresentation::OpenCL20: return createSPIRVToOCL20Legacy(); case SPIRV::BIsRepresentation::SPIRVFriendlyIR: // nothing to do, already done return nullptr; } llvm_unreachable("Unsupported built-ins representation"); return nullptr; } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToOCL.h000066400000000000000000000470401506646577400211060ustar00rootroot00000000000000//===- SPIRVToOCL.h - Converts SPIR-V to LLVM ------------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file contains declaration of SPIRVToOCL class which implements /// common transform of SPIR-V builtins to OCL builtins. /// //===----------------------------------------------------------------------===// #ifndef SPIRVTOOCL_H #define SPIRVTOOCL_H #include "OCLUtil.h" #include "SPIRVInternal.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include namespace SPIRV { class SPIRVToOCLBase : public InstVisitor { public: SPIRVToOCLBase() : M(nullptr), Ctx(nullptr) {} virtual ~SPIRVToOCLBase() {} virtual bool runSPIRVToOCL(Module &M) = 0; void visitCallInst(CallInst &CI); // SPIR-V reader should translate vector casts into OCL built-ins because // such conversions are not defined neither by OpenCL C/C++ nor // by SPIR 1.2/2.0 standards. So, it is safer to convert such casts into // appropriate calls to conversion built-ins defined by the standards. void visitCastInst(CastInst &CI); /// Transform __spirv_ImageQuerySize[Lod] into vector of the same length /// containing {[get_image_width | get_image_dim], get_image_array_size} /// for all images except image1d_t which is always converted into /// get_image_width returning scalar result. void visitCallSPRIVImageQuerySize(CallInst *CI); /// Transform __spirv_(NonUniform)Group* to {work_group|sub_group}_*. /// /// Special handling of work_group_broadcast. /// __spirv_GroupBroadcast(a, vec3(x, y, z)) /// => /// work_group_broadcast(a, x, y, z) /// /// Special handling of sub_group_all, sub_group_any, /// sub_group_non_uniform_all, sub_group_non_uniform_any, sub_group_ballot, /// sub_group_clustered_logical_[and/or/xor]. /// retTy func(i1 arg) /// => /// retTy func(i32 arg) /// /// Special handling of sub_group_all, sub_group_any, /// sub_group_non_uniform_all, /// sub_group_non_uniform_any, sub_group_non_uniform_all_equal. /// i1 func /// => /// i32 func void visitCallSPIRVGroupBuiltin(CallInst *CI, Op OC); /// Transform __spirv_{PipeOpName} to OCL pipe builtin functions. void visitCallSPIRVPipeBuiltin(CallInst *CI, Op OC); /// Transform __spirv_OpOpSubgroupImageMediaBlockReadINTEL => /// intel_sub_group_media_block_read /// __spirv_OpSubgroupImageMediaBlockWriteINTEL => /// intel_sub_group_media_block_write void visitCallSPIRVImageMediaBlockBuiltin(CallInst *CI, Op OC); /// Transform __spirv_OpGenericCastToPtrExplicit_To{Global|Local|Private} to /// to_{global|local|private} OCL builtin. void visitCallGenericCastToPtrExplicitBuiltIn(CallInst *CI, Op OC); /// Transform __spirv_OpBuildINDRange_{1|2|3}D to /// ndrange_{1|2|3}D OCL builtin. void visitCallBuildNDRangeBuiltIn(CallInst *CI, Op OC, StringRef DemangledName); /// Transform __spirv_*Convert_R{ReturnType}{_sat}{_rtp|_rtn|_rtz|_rte} to /// convert_{ReturnType}_{sat}{_rtp|_rtn|_rtz|_rte} /// example: <2 x i8> __spirv_SatConvertUToS(<2 x i32>) => /// convert_uchar2_sat(int2) void visitCallSPIRVCvtBuiltin(CallInst *CI, Op OC, StringRef DemangledName); /// Transform /// __spirv_AsyncGroupCopy(ScopeWorkGroup, dst, src, n, stride, event) /// => async_work_group_strided_copy(dst, src, n, stride, event) void visitCallAsyncWorkGroupCopy(CallInst *CI, Op OC); /// Transform __spirv_GroupWaitEvents(Scope, NumEvents, EventsList) /// => wait_group_events(NumEvents, EventsList) void visitCallGroupWaitEvents(CallInst *CI, Op OC); /// Transform __spirv_ImageSampleExplicitLod__{ReturnType} to read_imade void visitCallSPIRVImageSampleExplicitLodBuiltIn(CallInst *CI, Op OC); /// Transform __spirv_ImageWrite to write_image void visitCallSPIRVImageWriteBuiltIn(CallInst *CI, Op OC); /// Transform __spirv_ImageRead to read_image void visitCallSPIRVImageReadBuiltIn(CallInst *CI, Op OC); /// Transform __spirv_ImageQueryOrder to get_image_channel_order // __spirv_ImageQueryFormat to get_image_channel_data_type void visitCallSPIRVImageQueryBuiltIn(CallInst *CI, Op OC); /// Transform subgroup Intel opcodes /// example: __spirv_SubgroupBlockWriteINTEL /// => intel_sub_group_block_write_ul void visitCallSPIRVSubgroupINTELBuiltIn(CallInst *CI, Op OC); /// Transform AVC INTEL Evaluate opcodes /// example: __spirv_SubgroupAvcImeEvaluateWithSingleReference /// => intel_sub_group_avc_ime_evaluate_with_single_reference void visitCallSPIRVAvcINTELEvaluateBuiltIn(CallInst *CI, Op OC); /// Transform AVC INTEL general opcodes /// example: __spirv_SubgroupAvcMceGetDefaultInterBaseMultiReferencePenalty /// => /// intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty void visitCallSPIRVAvcINTELInstructionBuiltin(CallInst *CI, Op OC); /// Transform __spirv_GenericPtrMemSemantics to: /// %0 = call spirv_func i32 @_Z9get_fence /// %1 = shl i31 %0, 8 void visitCallSPIRVGenericPtrMemSemantics(CallInst *CI); /// Transform __spirv_ConvertFToBF16INTELDv(N)_f to: /// intel_convert_bfloat16(N)_as_ushort(N)Dv(N)_f; /// and transform __spirv_ConvertBF16ToFINTELDv(N)_s to: /// intel_convert_as_bfloat16(N)_float(N)Dv(N)_t; /// where N is vector size void visitCallSPIRVBFloat16Conversions(CallInst *CI, Op OC); /// Transform __spirv_* builtins to OCL 2.0 builtins. /// No change with arguments. void visitCallSPIRVBuiltin(CallInst *CI, Op OC); /// Transform __spirv_* builtins (originates from builtin variables) to OCL /// builtins. /// No change with arguments. /// e.g. /// _Z33__spirv_BuiltInGlobalInvocationIdi(x) -> get_global_id(x) void visitCallSPIRVBuiltin(CallInst *CI, SPIRVBuiltinVariableKind Kind); /// Transform __spirv_ocl* instructions (OpenCL Extended Instruction Set) /// to OpenCL builtins. void visitCallSPIRVOCLExt(CallInst *CI, OCLExtOpKind Kind); /// Transform __spirv_ocl_vstore* to corresponding vstore OpenCL instruction void visitCallSPIRVVStore(CallInst *CI, OCLExtOpKind Kind); /// Transform __spirv_ocl_vloadn to OpenCL vload[2|4|8|16] void visitCallSPIRVVLoadn(CallInst *CI, OCLExtOpKind Kind); /// Transform __spirv_ocl_printf to (i8 addrspace(2)*, ...) @printf void visitCallSPIRVPrintf(CallInst *CI, OCLExtOpKind Kind); /// Get prefix work_/sub_ for OCL group builtin functions. /// Assuming the first argument of \param CI is a constant integer for /// workgroup/subgroup scope enums. std::string getGroupBuiltinPrefix(CallInst *CI); /// Transform __spirv_OpAtomicCompareExchange and /// __spirv_OpAtomicCompareExchangeWeak virtual Instruction *visitCallSPIRVAtomicCmpExchg(CallInst *CI) = 0; /// Transform __spirv_OpAtomicIIncrement/OpAtomicIDecrement to: /// - OCL2.0: atomic_fetch_add_explicit/atomic_fetch_sub_explicit /// - OCL1.2: atomic_inc/atomic_dec virtual Instruction *visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) = 0; /// Transform __spirv_Atomic* to atomic_*. /// __spirv_Atomic*(atomic_op, scope, sema, ops, ...) => /// atomic_*(atomic_op, ops, ..., order(sema), map(scope)) virtual Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) = 0; /// Transform __spirv_MemoryBarrier to: /// - OCL2.0: atomic_work_item_fence.__spirv_MemoryBarrier(scope, sema) => /// atomic_work_item_fence(flag(sema), order(sema), map(scope)) /// - OCL1.2: mem_fence virtual void visitCallSPIRVMemoryBarrier(CallInst *CI) = 0; /// Transform __spirv_ControlBarrier to: /// - OCL2.0: work_group_barrier or sub_group barrier /// - OCL1.2: barrier virtual void visitCallSPIRVControlBarrier(CallInst *CI) = 0; /// Transform split __spirv_ControlBarrierArriveINTEL and /// __spirv_ControlBarrierWaitINTEL barrier to: /// - OCL2.0: overload with a memory_scope argument /// - OCL1.2: overload with no memory_scope argument virtual void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) = 0; /// Transform __spirv_EnqueueKernel to __enqueue_kernel virtual void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) = 0; /// Transform __spirv_Any and __spirv_All to OpenCL builtin. void visitCallSPIRVAnyAll(CallInst *CI, Op OC); /// Transform relational builtin, e.g. __spirv_IsNan, to OpenCL builtin. void visitCallSPIRVRelational(CallInst *CI, Op OC); /// Transform __spirv_ReadClockKHR to OpenCL builtin. void visitCallSPIRVReadClockKHR(CallInst *CI); /// Conduct generic mutations for all atomic builtins virtual CallInst *mutateCommonAtomicArguments(CallInst *CI, Op OC) = 0; /// Transform __spirv_Opcode to ocl-version specific builtin name /// using separate maps for OpenCL 1.2 and OpenCL 2.0 virtual Instruction *mutateAtomicName(CallInst *CI, Op OC) = 0; // Transform FP atomic opcode to corresponding OpenCL function name virtual std::string mapFPAtomicName(Op OC) = 0; void translateOpaqueTypes(); private: /// Transform uniform group opcode to corresponding OpenCL function name, /// example: GroupIAdd(Reduce) => group_iadd => work_group_reduce_add | /// sub_group_reduce_add std::string getUniformArithmeticBuiltinName(CallInst *CI, Op OC); /// Transform non-uniform group opcode to corresponding OpenCL function name, /// example: GroupNonUniformIAdd(Reduce) => group_non_uniform_iadd => /// sub_group_non_uniform_reduce_add std::string getNonUniformArithmeticBuiltinName(CallInst *CI, Op OC); /// Transform ballot bit count opcode to corresponding OpenCL function name, /// example: GroupNonUniformBallotBitCount(Reduce) => /// group_ballot_bit_count_iadd => sub_group_ballot_bit_count std::string getBallotBuiltinName(CallInst *CI, Op OC); /// Transform OpGroupNonUniformRotateKHR to corresponding OpenCL function /// name. std::string getRotateBuiltinName(CallInst *CI, Op OC); /// Transform group opcode to corresponding OpenCL function name std::string groupOCToOCLBuiltinName(CallInst *CI, Op OC); /// Transform SPV-IR image opaque type into OpenCL representation, /// example: spirv.Image._void_1_0_0_0_0_0_1 => opencl.image2d_wo_t std::string getOCLImageOpaqueType(SmallVector &Postfixes); /// Transform SPV-IR pipe opaque type into OpenCL representation, /// example: spirv.Pipe._0 => opencl.pipe_ro_t std::string getOCLPipeOpaqueType(SmallVector &Postfixes); void getParameterTypes(CallInst *CI, SmallVectorImpl &Tys); std::string translateOpaqueType(StringRef STName); /// Mutate the argument list based on (optional) image operands at position /// ImOpArgIndex. Set IsSigned according to any SignExtend/ZeroExtend Image /// Operands present in Args, or default to signed if there are none. void mutateArgsForImageOperands(std::vector &Args, unsigned ImOpArgIndex, bool &IsSigned); protected: Module *M; LLVMContext *Ctx; }; class SPIRVToOCLLegacy : public ModulePass { protected: SPIRVToOCLLegacy(char &ID) : ModulePass(ID) {} public: bool runOnModule(Module &M) override = 0; }; class SPIRVToOCL12Base : public SPIRVToOCLBase { public: bool runSPIRVToOCL(Module &M) override; /// Transform __spirv_MemoryBarrier to atomic_work_item_fence. /// __spirv_MemoryBarrier(scope, sema) => /// atomic_work_item_fence(flag(sema), order(sema), map(scope)) void visitCallSPIRVMemoryBarrier(CallInst *CI) override; /// Transform __spirv_ControlBarrier to barrier. /// __spirv_ControlBarrier(execScope, memScope, sema) => /// barrier(flag(sema)) void visitCallSPIRVControlBarrier(CallInst *CI) override; /// Transform split __spirv_ControlBarrierArriveINTEL and /// __spirv_ControlBarrierWaitINTEL barrier to overloads without a /// memory_scope argument. void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override; /// Transform __spirv_OpAtomic functions. It firstly conduct generic /// mutations for all builtins and then mutate some of them seperately Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) override; /// Transform __spirv_OpAtomicIIncrement / OpAtomicIDecrement to /// atomic_inc / atomic_dec Instruction *visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) override; /// Transform __spirv_OpAtomicUMin/SMin/UMax/SMax into /// atomic_min/atomic_max, as there is no distinction in OpenCL 1.2 /// between signed and unsigned version of those functions Instruction *visitCallSPIRVAtomicUMinUMax(CallInst *CI, Op OC); /// Transform __spirv_OpAtomicLoad to atomic_add(*ptr, 0) Instruction *visitCallSPIRVAtomicLoad(CallInst *CI); /// Transform __spirv_OpAtomicStore to atomic_xchg(*ptr, value) Instruction *visitCallSPIRVAtomicStore(CallInst *CI); /// Transform __spirv_OpAtomicFlagClear to atomic_xchg(*ptr, 0) /// with ignoring the result Instruction *visitCallSPIRVAtomicFlagClear(CallInst *CI); /// Transform __spirv_OpAtomicFlagTestAndTest to /// (bool)atomic_xchg(*ptr, 1) Instruction *visitCallSPIRVAtomicFlagTestAndSet(CallInst *CI); /// Transform __spirv_OpAtomicCompareExchange/Weak into atomic_cmpxchg /// OpAtomicCompareExchangeWeak is not "weak" at all, but instead has /// the same semantics as OpAtomicCompareExchange. Instruction *visitCallSPIRVAtomicCmpExchg(CallInst *CI) override; /// Trigger assert, since OpenCL 1.2 doesn't support enqueue_kernel void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) override; /// Conduct generic mutations for all atomic builtins CallInst *mutateCommonAtomicArguments(CallInst *CI, Op OC) override; /// Transform atomic builtin name into correct ocl-dependent name Instruction *mutateAtomicName(CallInst *CI, Op OC) override; // Transform FP atomic opcode to corresponding OpenCL function name std::string mapFPAtomicName(Op OC) override; /// Transform SPIR-V atomic instruction opcode into OpenCL 1.2 builtin name. /// Depending on the type, the return name starts with "atomic_" for 32-bit /// types or with "atom_" for 64-bit types, as specified by /// cl_khr_int64_base_atomics and cl_khr_int64_extended_atomics extensions. std::string mapAtomicName(Op OC, Type *Ty); }; class SPIRVToOCL12Pass : public llvm::PassInfoMixin, public SPIRVToOCL12Base { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runSPIRVToOCL(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } }; class SPIRVToOCL12Legacy : public SPIRVToOCL12Base, public SPIRVToOCLLegacy { public: SPIRVToOCL12Legacy() : SPIRVToOCLLegacy(ID) { initializeSPIRVToOCL12LegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override; static char ID; }; class SPIRVToOCL20Base : public SPIRVToOCLBase { public: bool runSPIRVToOCL(Module &M) override; /// Transform __spirv_MemoryBarrier to atomic_work_item_fence. /// __spirv_MemoryBarrier(scope, sema) => /// atomic_work_item_fence(flag(sema), order(sema), map(scope)) void visitCallSPIRVMemoryBarrier(CallInst *CI) override; /// Transform __spirv_ControlBarrier to work_group_barrier/sub_group_barrier. /// If execution scope is ScopeWorkgroup: /// __spirv_ControlBarrier(execScope, memScope, sema) => /// work_group_barrier(flag(sema), map(memScope)) /// Otherwise: /// __spirv_ControlBarrier(execScope, memScope, sema) => /// sub_group_barrier(flag(sema), map(memScope)) void visitCallSPIRVControlBarrier(CallInst *CI) override; /// Transform split __spirv_ControlBarrierArriveINTEL and /// __spirv_ControlBarrierWaitINTEL barrier to overloads with a /// memory_scope argument. void visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) override; /// Transform __spirv_Atomic* to atomic_*. /// __spirv_Atomic*(atomic_op, scope, sema, ops, ...) => /// atomic_*(generic atomic_op, ops, ..., order(sema), map(scope)) Instruction *visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) override; /// Transform __spirv_OpAtomicIIncrement / OpAtomicIDecrement to /// atomic_fetch_add_explicit / atomic_fetch_sub_explicit Instruction *visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) override; /// Transform __spirv_EnqueueKernel to __enqueue_kernel void visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) override; /// Conduct generic mutations for all atomic builtins CallInst *mutateCommonAtomicArguments(CallInst *CI, Op OC) override; /// Transform atomic builtin name into correct ocl-dependent name Instruction *mutateAtomicName(CallInst *CI, Op OC) override; // Transform FP atomic opcode to corresponding OpenCL function name std::string mapFPAtomicName(Op OC) override; /// Transform __spirv_OpAtomicCompareExchange/Weak into /// atomic_compare_exchange_strong_explicit /// OpAtomicCompareExchangeWeak is not "weak" at all, but instead has /// the same semantics as OpAtomicCompareExchange. Instruction *visitCallSPIRVAtomicCmpExchg(CallInst *CI) override; }; class SPIRVToOCL20Pass : public llvm::PassInfoMixin, public SPIRVToOCL20Base { public: llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { return runSPIRVToOCL(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } }; class SPIRVToOCL20Legacy : public SPIRVToOCLLegacy, public SPIRVToOCL20Base { public: SPIRVToOCL20Legacy() : SPIRVToOCLLegacy(ID) { initializeSPIRVToOCL20LegacyPass(*PassRegistry::getPassRegistry()); } bool runOnModule(Module &M) override; static char ID; }; /// Add passes for translating SPIR-V Instructions to the desired /// representation in LLVM IR (such as OpenCL builtins or SPIR-V Friendly IR). void addSPIRVBIsLoweringPass(ModulePassManager &PassMgr, SPIRV::BIsRepresentation BIsRep); } // namespace SPIRV #endif // SPIRVTOOCL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToOCL12.cpp000066400000000000000000000260141506646577400216020ustar00rootroot00000000000000//===- SPIRVToOCL12.cpp - Transform SPIR-V builtins to OCL 1.2 // builtins------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements transform of SPIR-V builtins to OCL 1.2 builtins. // //===----------------------------------------------------------------------===// #include "SPIRVToOCL.h" #include "llvm/IR/Verifier.h" #define DEBUG_TYPE "spvtocl12" namespace SPIRV { char SPIRVToOCL12Legacy::ID = 0; bool SPIRVToOCL12Legacy::runOnModule(Module &Module) { return SPIRVToOCL12Base::runSPIRVToOCL(Module); } bool SPIRVToOCL12Base::runSPIRVToOCL(Module &Module) { M = &Module; Ctx = &M->getContext(); // Lower builtin variables to builtin calls first. lowerBuiltinVariablesToCalls(M); translateOpaqueTypes(); visit(*M); postProcessBuiltinsReturningStruct(M); postProcessBuiltinsWithArrayArguments(M); eraseUselessFunctions(&Module); LLVM_DEBUG(dbgs() << "After SPIRVToOCL12:\n" << *M); std::string Err; raw_string_ostream ErrorOS(Err); if (verifyModule(*M, &ErrorOS)) { LLVM_DEBUG(errs() << "Fails to verify module: " << ErrorOS.str()); } return true; } void SPIRVToOCL12Base::visitCallSPIRVMemoryBarrier(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Value *MemFenceFlags = transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[1], CI); Args.assign(1, MemFenceFlags); return kOCLBuiltinName::MemFence; }, &Attrs); } void SPIRVToOCL12Base::visitCallSPIRVControlBarrier(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { auto *MemFenceFlags = transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI); Args.assign(1, MemFenceFlags); return kOCLBuiltinName::Barrier; }, &Attrs); } void SPIRVToOCL12Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Value *MemFenceFlags = SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI); Args.assign(1, MemFenceFlags); return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(1); return mapAtomicName(OC, CI->getType()); }, &Attrs); } CallInst *SPIRVToOCL12Base::mutateCommonAtomicArguments(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { auto Ptr = findFirstPtr(Args); auto NumOrder = getSPIRVAtomicBuiltinNumMemoryOrderArgs(OC); auto ArgsToRemove = NumOrder + 1; // OpenCL1.2 builtins does not use // scope and memory order arguments auto StartIdx = Ptr + 1; auto StopIdx = StartIdx + ArgsToRemove; Args.erase(Args.begin() + StartIdx, Args.begin() + StopIdx); return mapAtomicName(OC, CI->getType()); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicUMinUMax(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { std::swap(Args[1], Args[3]); Args.resize(2); return mapAtomicName(OC, CI->getType()); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicLoad(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Args.resize(1); // There is no atomic_load in OpenCL 1.2 spec. // Emit this builtin via call of atomic_add(*p, 0). Type *PtrElemTy = CI->getType(); Args.push_back(Constant::getNullValue(PtrElemTy)); return mapAtomicName(OpAtomicIAdd, PtrElemTy); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicStore(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { std::swap(Args[1], Args[3]); Args.resize(2); // The type of the value pointed to by Pointer (1st argument), or the // value being exchanged (2nd argument) must be the same as Result Type. RetTy = Args[1]->getType(); return mapAtomicName(OpAtomicExchange, RetTy); }, [=](CallInst *CI) -> Instruction * { return CI; }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicFlagClear(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { Args.resize(1); Args.push_back(getInt32(M, 0)); RetTy = Type::getInt32Ty(M->getContext()); return mapAtomicName(OpAtomicExchange, RetTy); }, [=](CallInst *CI) -> Instruction * { return CI; }, &Attrs); } Instruction * SPIRVToOCL12Base::visitCallSPIRVAtomicFlagTestAndSet(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { Args.resize(1); Args.push_back(getInt32(M, 1)); RetTy = Type::getInt32Ty(M->getContext()); return mapAtomicName(OpAtomicExchange, RetTy); }, [=](CallInst *CI) -> Instruction * { return BitCastInst::Create(Instruction::Trunc, CI, Type::getInt1Ty(CI->getContext()), "", CI->getNextNode()); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicCmpExchg(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Args.erase(Args.begin() + 1, Args.begin() + 4); // SPIRV OpAtomicCompareExchange and // OpAtomicCompareExchangeWeak has Value and // Comparator in different order than ocl functions // both of them are translated into atomic_cmpxchg std::swap(Args[1], Args[2]); // Type of return value, pointee of the pointer // operand, other operands, all match, and should // be integer scalar types. return mapAtomicName(OpAtomicCompareExchange, CI->getType()); }, &Attrs); } Instruction *SPIRVToOCL12Base::visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) { Instruction *NewCI = nullptr; switch (OC) { case OpAtomicLoad: NewCI = visitCallSPIRVAtomicLoad(CI); break; case OpAtomicStore: NewCI = visitCallSPIRVAtomicStore(CI); break; case OpAtomicFlagClear: NewCI = visitCallSPIRVAtomicFlagClear(CI); break; case OpAtomicFlagTestAndSet: NewCI = visitCallSPIRVAtomicFlagTestAndSet(CI); break; case OpAtomicUMin: case OpAtomicUMax: NewCI = visitCallSPIRVAtomicUMinUMax(CI, OC); break; case OpAtomicCompareExchange: case OpAtomicCompareExchangeWeak: NewCI = visitCallSPIRVAtomicCmpExchg(CI); break; default: NewCI = mutateCommonAtomicArguments(CI, OC); } return NewCI; } void SPIRVToOCL12Base::visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) { assert(0 && "OpenCL 1.2 doesn't support enqueue_kernel!"); } std::string SPIRVToOCL12Base::mapFPAtomicName(Op OC) { assert(isFPAtomicOpCode(OC) && "Not intended to handle other opcodes than " "AtomicF{Add/Min/Max}EXT!"); switch (OC) { case OpAtomicFAddEXT: return "atomic_add"; case OpAtomicFMinEXT: return "atomic_min"; case OpAtomicFMaxEXT: return "atomic_max"; default: llvm_unreachable("Unsupported opcode!"); } } Instruction *SPIRVToOCL12Base::mutateAtomicName(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { return OCL12SPIRVBuiltinMap::rmap(OC); }, &Attrs); } std::string SPIRVToOCL12Base::mapAtomicName(Op OC, Type *Ty) { std::string Prefix = Ty->isIntegerTy(64) ? kOCLBuiltinName::AtomPrefix : kOCLBuiltinName::AtomicPrefix; // Map fp atomic instructions to regular OpenCL built-ins. if (isFPAtomicOpCode(OC)) return mapFPAtomicName(OC); return Prefix += OCL12SPIRVBuiltinMap::rmap(OC); } } // namespace SPIRV INITIALIZE_PASS(SPIRVToOCL12Legacy, "spvtoocl12", "Translate SPIR-V builtins to OCL 1.2 builtins", false, false) ModulePass *llvm::createSPIRVToOCL12Legacy() { return new SPIRVToOCL12Legacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVToOCL20.cpp000066400000000000000000000345311506646577400216040ustar00rootroot00000000000000//===- SPIRVToOCL20.cpp - Transform SPIR-V builtins to OCL20 builtins------===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements transform SPIR-V builtins to OCL 2.0 builtins. // //===----------------------------------------------------------------------===// #define DEBUG_TYPE "spvtocl20" #include "OCLUtil.h" #include "SPIRVToOCL.h" #include "llvm/IR/Verifier.h" namespace SPIRV { char SPIRVToOCL20Legacy::ID = 0; bool SPIRVToOCL20Legacy::runOnModule(Module &Module) { return SPIRVToOCL20Base::runSPIRVToOCL(Module); } bool SPIRVToOCL20Base::runSPIRVToOCL(Module &Module) { M = &Module; Ctx = &M->getContext(); // Lower builtin variables to builtin calls first. lowerBuiltinVariablesToCalls(M); translateOpaqueTypes(); visit(*M); postProcessBuiltinsReturningStruct(M); postProcessBuiltinsWithArrayArguments(M); eraseUselessFunctions(&Module); LLVM_DEBUG(dbgs() << "After SPIRVToOCL20:\n" << *M); std::string Err; raw_string_ostream ErrorOS(Err); if (verifyModule(*M, &ErrorOS)) { LLVM_DEBUG(errs() << "Fails to verify module: " << ErrorOS.str()); } return true; } void SPIRVToOCL20Base::visitCallSPIRVMemoryBarrier(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { Value *MemScope = SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Args[0], CI); Value *MemFenceFlags = SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[1], CI); Value *MemOrder = SPIRV::transSPIRVMemorySemanticsIntoOCLMemoryOrder(Args[1], CI); Args.resize(3); Args[0] = MemFenceFlags; Args[1] = MemOrder; Args[2] = MemScope; return kOCLBuiltinName::AtomicWorkItemFence; }, &Attrs); } void SPIRVToOCL20Base::visitCallSPIRVControlBarrier(CallInst *CI) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); SmallVector ArgAttrs = {Attrs.getParamAttrs(1), Attrs.getParamAttrs(2)}; AttributeList NewAttrs = AttributeList::get(*Ctx, Attrs.getFnAttrs(), Attrs.getRetAttrs(), ArgAttrs); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { auto GetArg = [=](unsigned I) { return cast(Args[I])->getZExtValue(); }; auto ExecScope = static_cast(GetArg(0)); Value *MemScope = getInt32(M, rmap(static_cast(GetArg(1)))); Value *MemFenceFlags = SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI); Args.resize(2); Args[0] = MemFenceFlags; Args[1] = MemScope; return (ExecScope == ScopeWorkgroup) ? kOCLBuiltinName::WorkGroupBarrier : kOCLBuiltinName::SubGroupBarrier; }, &NewAttrs); } void SPIRVToOCL20Base::visitCallSPIRVSplitBarrierINTEL(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { auto GetArg = [=](unsigned I) { return cast(Args[I])->getZExtValue(); }; Value *MemScope = getInt32(M, rmap(static_cast(GetArg(1)))); Value *MemFenceFlags = SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(Args[2], CI); Args.resize(2); Args[0] = MemFenceFlags; Args[1] = MemScope; return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } std::string SPIRVToOCL20Base::mapFPAtomicName(Op OC) { assert(isFPAtomicOpCode(OC) && "Not intended to handle other opcodes than " "AtomicF{Add/Min/Max}EXT!"); switch (OC) { case OpAtomicFAddEXT: return "atomic_fetch_add_explicit"; case OpAtomicFMinEXT: return "atomic_fetch_min_explicit"; case OpAtomicFMaxEXT: return "atomic_fetch_max_explicit"; default: llvm_unreachable("Unsupported opcode!"); } } Instruction *SPIRVToOCL20Base::mutateAtomicName(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // Map fp atomic instructions to regular OpenCL built-ins. if (isFPAtomicOpCode(OC)) return mapFPAtomicName(OC); return OCLSPIRVBuiltinMap::rmap(OC); }, &Attrs); } Instruction *SPIRVToOCL20Base::visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) { CallInst *CIG = mutateCommonAtomicArguments(CI, OC); Instruction *NewCI = nullptr; switch (OC) { case OpAtomicIIncrement: case OpAtomicIDecrement: NewCI = visitCallSPIRVAtomicIncDec(CIG, OC); break; case OpAtomicCompareExchange: case OpAtomicCompareExchangeWeak: NewCI = visitCallSPIRVAtomicCmpExchg(CIG); break; default: NewCI = mutateAtomicName(CIG, OC); } return NewCI; } Instruction *SPIRVToOCL20Base::visitCallSPIRVAtomicIncDec(CallInst *CI, Op OC) { AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { // Since OpenCL 2.0 doesn't have atomic_inc and atomic_dec builtins, // we translate these instructions to atomic_fetch_add_explicit and // atomic_fetch_sub_explicit OpenCL 2.0 builtins with "operand" argument // = 1. auto Name = OCLSPIRVBuiltinMap::rmap( OC == OpAtomicIIncrement ? OpAtomicIAdd : OpAtomicISub); Type *ValueTy = CI->getType(); assert(ValueTy->isIntegerTy()); Args.insert(Args.begin() + 1, llvm::ConstantInt::get(ValueTy, 1)); return Name; }, &Attrs); } CallInst *SPIRVToOCL20Base::mutateCommonAtomicArguments(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { for (size_t I = 0; I < Args.size(); ++I) { Value *PtrArg = Args[I]; Type *PtrArgTy = PtrArg->getType(); if (PtrArgTy->isPointerTy()) { if (PtrArgTy->getPointerAddressSpace() != SPIRAS_Generic) { Type *FixedPtr = PointerType::getWithSamePointeeType( cast(PtrArgTy), SPIRAS_Generic); Args[I] = CastInst::CreatePointerBitCastOrAddrSpaceCast( PtrArg, FixedPtr, PtrArg->getName() + ".as", CI); } } } auto Ptr = findFirstPtr(Args); std::string Name; // Map fp atomic instructions to regular OpenCL built-ins. if (isFPAtomicOpCode(OC)) Name = mapFPAtomicName(OC); else Name = OCLSPIRVBuiltinMap::rmap(OC); auto NumOrder = getSPIRVAtomicBuiltinNumMemoryOrderArgs(OC); auto ScopeIdx = Ptr + 1; auto OrderIdx = Ptr + 2; Args[ScopeIdx] = SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Args[ScopeIdx], CI); for (size_t I = 0; I < NumOrder; ++I) { Args[OrderIdx + I] = SPIRV::transSPIRVMemorySemanticsIntoOCLMemoryOrder( Args[OrderIdx + I], CI); } std::swap(Args[ScopeIdx], Args.back()); return Name; }, &Attrs); } Instruction *SPIRVToOCL20Base::visitCallSPIRVAtomicCmpExchg(CallInst *CI) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Instruction *PInsertBefore = CI; Type *MemTy = CI->getType(); return mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args, Type *&RetTy) { // OpAtomicCompareExchange[Weak] semantics is different from // atomic_compare_exchange_strong semantics as well as // arguments order. // OCL built-ins returns boolean value and stores a new/original // value by pointer passed as 2nd argument (aka expected) while SPIR-V // instructions returns this new/original value as a resulting value. AllocaInst *PExpected = new AllocaInst(MemTy, 0, "expected", &(*PInsertBefore->getParent() ->getParent() ->getEntryBlock() .getFirstInsertionPt())); PExpected->setAlignment(Align(MemTy->getScalarSizeInBits() / 8)); new StoreInst(Args[1], PExpected, PInsertBefore); unsigned AddrSpc = SPIRAS_Generic; Type *PtrTyAS = PointerType::getWithSamePointeeType( cast(PExpected->getType()), AddrSpc); Args[1] = CastInst::CreatePointerBitCastOrAddrSpaceCast( PExpected, PtrTyAS, PExpected->getName() + ".as", PInsertBefore); std::swap(Args[3], Args[4]); std::swap(Args[2], Args[3]); RetTy = Type::getInt1Ty(*Ctx); // OpAtomicCompareExchangeWeak is not "weak" at all, but instead has // the same semantics as OpAtomicCompareExchange. return "atomic_compare_exchange_strong_explicit"; }, [=](CallInst *CI) -> Instruction * { // OCL built-ins atomic_compare_exchange_[strong|weak] return boolean // value. So, to obtain the same value as SPIR-V instruction is // returning it has to be loaded from the memory where 'expected' // value is stored. This memory must contain the needed value after a // call to OCL built-in is completed. return new LoadInst(MemTy, CI->getArgOperand(1), "original", PInsertBefore); }, &Attrs); } void SPIRVToOCL20Base::visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) { assert(CI->getCalledFunction() && "Unexpected indirect call"); AttributeList Attrs = CI->getCalledFunction()->getAttributes(); Instruction *PInsertBefore = CI; mutateCallInstOCL( M, CI, [=](CallInst *, std::vector &Args) { bool HasVaargs = Args.size() > 10; bool HasEvents = true; Value *EventRet = Args[5]; if (isa(EventRet)) { Value *NumEvents = Args[3]; if (isa(NumEvents)) { ConstantInt *NE = cast(NumEvents); HasEvents = NE->getZExtValue() != 0; } } Value *Invoke = Args[6]; auto *Int8PtrTyGen = Type::getInt8PtrTy(*Ctx, SPIRAS_Generic); Args[6] = CastInst::CreatePointerBitCastOrAddrSpaceCast( Invoke, Int8PtrTyGen, "", PInsertBefore); // Don't remove arguments immediately, just mark them as removed with // nullptr, and remove them at the end of processing. It allows for // easier understanding of which argument is going to be removed. auto MarkAsRemoved = [&Args](size_t Start, size_t End) { assert(Start <= End); for (size_t I = Start; I < End; I++) Args[I] = nullptr; }; if (!HasEvents) { // Mark arguments at indices 3 (Num Events), 4 (Wait Events), 5 (Ret // Event) as removed. MarkAsRemoved(3, 6); } if (!HasVaargs) { // Mark arguments at indices 8 (Param Size), 9 (Param Align) as // removed. MarkAsRemoved(8, 10); } else { // GEP to array of sizes of local arguments Value *GEP = Args[10]; size_t NumLocalArgs = Args.size() - 10; // Mark all SPIRV-specific arguments as removed MarkAsRemoved(8, Args.size()); Type *Int32Ty = Type::getInt32Ty(*Ctx); Args[8] = ConstantInt::get(Int32Ty, NumLocalArgs); Args[9] = GEP; } Args.erase(std::remove(Args.begin(), Args.end(), nullptr), Args.end()); std::string FName = ""; if (!HasVaargs && !HasEvents) FName = "__enqueue_kernel_basic"; else if (!HasVaargs && HasEvents) FName = "__enqueue_kernel_basic_events"; else if (HasVaargs && !HasEvents) FName = "__enqueue_kernel_varargs"; else FName = "__enqueue_kernel_events_varargs"; return FName; }, &Attrs); } } // namespace SPIRV INITIALIZE_PASS(SPIRVToOCL20Legacy, "spvtoocl20", "Translate SPIR-V builtins to OCL 2.0 builtins", false, false) ModulePass *llvm::createSPIRVToOCL20Legacy() { return new SPIRVToOCL20Legacy(); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVTypeScavenger.cpp000066400000000000000000000530021506646577400232730ustar00rootroot00000000000000//===- SPIRVTypeScavenger.cpp - Recover pointer types in opaque pointer IR ===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements the necessary logic to recover pointer types from LLVM // IR for the output SPIR-V file after LLVM IR completes its transition to // opaque pointers. // //===----------------------------------------------------------------------===// #include "SPIRVTypeScavenger.h" #include "SPIRVInternal.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/NoFolder.h" #include "llvm/IR/Operator.h" #define DEBUG_TYPE "type-scavenger" using namespace llvm; void SPIRVTypeScavenger::typeModule(Module &M) { // If typed pointers are in effect, we need to do nothing here. if (M.getContext().supportsTypedPointers()) return; // Try to fill in any known types for function parameters. for (auto &F : M.functions()) { deduceFunctionType(F); } // Collect types for all pertinent values in the module. for (auto &F : M.functions()) { for (Argument &Arg : F.args()) if (Arg.getType()->isPointerTy()) computePointerElementType(&Arg); for (BasicBlock &BB : F) { for (Instruction &I : BB) { if (I.getType()->isPointerTy()) computePointerElementType(&I); correctUseTypes(I); } } } // Go through all of the types we have collected, and if any are still // deferred, assign them a fallback i8* type. Type *Int8Ty = Type::getInt8Ty(M.getContext()); for (const auto &Pair : DeducedTypes) { if (auto *Untyped = dyn_cast(Pair.second)) { LLVM_DEBUG(dbgs() << "No inferrable type for " << *Pair.first << "\n"); fixType(*Untyped, Int8Ty); DeducedTypes[Pair.first] = Int8Ty; } } return; } static Type *getPointerUseType(Function *F, Op Opcode, unsigned ArgNo) { switch (Opcode) { case OpAtomicLoad: case OpAtomicExchange: case OpAtomicCompareExchange: case OpAtomicIAdd: case OpAtomicISub: case OpAtomicFAddEXT: case OpAtomicSMin: case OpAtomicUMin: case OpAtomicFMinEXT: case OpAtomicSMax: case OpAtomicUMax: case OpAtomicFMaxEXT: case OpAtomicAnd: case OpAtomicOr: case OpAtomicXor: if (ArgNo == 0) return F->getReturnType(); return nullptr; case OpAtomicStore: if (ArgNo == 0) return F->getArg(3)->getType(); return nullptr; default: return nullptr; } } void SPIRVTypeScavenger::deduceIntrinsicTypes(Function &F, Intrinsic::ID Id) { static constexpr unsigned Return = ~0U; auto AddParameter = [&](unsigned ArgNo, DeducedType Ty) { if (ArgNo == Return) { // TODO: Handle return types properly. } else { Argument *Arg = F.getArg(ArgNo); LLVM_DEBUG(dbgs() << "Parameter " << *Arg << " of " << F.getName() << " has type " << Ty << "\n"); DeducedTypes[Arg] = Ty; } }; LLVMContext &Ctx = F.getContext(); switch (Id) { case Intrinsic::memcpy: // First parameter is a pointer, but it may be any pointer type. return; case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: case Intrinsic::invariant_start: case Intrinsic::invariant_end: AddParameter(1, Type::getInt8Ty(Ctx)); return; // Second and third parameters are strings, which mean nothing. case Intrinsic::annotation: return; case Intrinsic::var_annotation: case Intrinsic::ptr_annotation: AddParameter(0, Type::getInt8Ty(Ctx)); // Second and third parameters are strings, so they can be any type. return; case Intrinsic::stacksave: AddParameter(Return, Type::getInt8Ty(Ctx)); return; case Intrinsic::stackrestore: AddParameter(0, Type::getInt8Ty(Ctx)); return; // llvm.instrprof.* intrinsics are not supported case Intrinsic::instrprof_cover: case Intrinsic::instrprof_increment: case Intrinsic::instrprof_increment_step: case Intrinsic::instrprof_value_profile: AddParameter(0, Type::getInt8Ty(Ctx)); return; } } static Type *getParamType(const AttributeList &AL, unsigned ArgNo) { if (Type *Ty = AL.getParamByValType(ArgNo)) return Ty; if (Type *Ty = AL.getParamStructRetType(ArgNo)) return Ty; if (Type *Ty = AL.getParamElementType(ArgNo)) return Ty; if (Type *Ty = AL.getParamInAllocaType(ArgNo)) return Ty; if (Type *Ty = AL.getParamPreallocatedType(ArgNo)) return Ty; return nullptr; } void SPIRVTypeScavenger::deduceFunctionType(Function &F) { SmallVector PointerArgs; for (Argument &Arg : F.args()) { if (Arg.getType()->isPointerTy()) PointerArgs.push_back(&Arg); } // Get any arguments from attributes where possible. for (Argument *Arg : PointerArgs) { Type *Ty = getParamType(F.getAttributes(), Arg->getArgNo()); if (Ty) DeducedTypes[Arg] = Ty; } // At this point, anything that we can get definitively correct is going to // come from declarations of builtins. If we have the actual implementation of // the function available, we should try to recover types from the function // definition itself. By early returning here, we ensure that remaining // arguments will get deferred types that will follow the regular typing // process. if (!F.isDeclaration()) return; // Recover known information from known SPIR-V builtin operations represented // as functions. StringRef DemangledName; if (oclIsBuiltin(F.getName(), DemangledName) || isDecoratedSPIRVFunc(&F, DemangledName)) { Op OC = getSPIRVFuncOC(DemangledName); if (OC != OpNop) { for (Argument *Arg : PointerArgs) { Type *PointeeTy = getPointerUseType(&F, OC, Arg->getArgNo()); if (PointeeTy) { DeducedTypes[Arg] = PointeeTy; LLVM_DEBUG(dbgs() << "Arg " << Arg->getArgNo() << " of " << F.getName() << " has type " << *PointeeTy << "\n"); } } } } if (auto IntrinID = F.getIntrinsicID()) { deduceIntrinsicTypes(F, IntrinID); } // If the function is a mangled name, try to recover types from the Itanium // name mangling. if (F.getName().startswith("_Z")) { SmallVector ParameterTypes; getParameterTypes(&F, ParameterTypes); for (Argument *Arg : PointerArgs) { if (auto *Ty = ParameterTypes[Arg->getArgNo()]) { DeducedTypes[Arg] = Ty; LLVM_DEBUG(dbgs() << "Arg " << Arg->getArgNo() << " of " << F.getName() << " has type " << *Ty << "\n"); } } } } /// Certain constant types (null, undef, and poison) will get their type from /// the use of the constant. We discover the type of the use by inserting a /// synthetic bitcast instruction before the use. For these types, we need to /// have special handling in a few places, and this indicates that it needs to /// be done. static bool doesNotImplyType(Value *V) { return isa(V) || isa(V); } SPIRVTypeScavenger::DeducedType SPIRVTypeScavenger::computePointerElementType(Value *V) { assert(V->getType()->isPointerTy() && "Trying to get the pointer type of a non-pointer value?"); // Don't try to store null, undef, or poison in our type map. We'll call these // i8* by default; if any use has a different type, a bitcast will be added // later. if (doesNotImplyType(V)) { return Type::getInt8Ty(V->getContext()); } // Check if we've already deduced a type for the value. DeducedType &Ty = DeducedTypes[V]; if (Ty) { return Ty; } // There are basically three categories of pointer-typed values: // 1. Values that have a well-defined pointee type (e.g., alloca). Return the // known type from this method. // 2. Values that have no intrinsic type (e.g., inttoptr). A new deferred // type construct will be created that will allow a use to identify the // type instead. // 3. Values that propagate their source type (e.g., phi). This wil return // the type of their source argument, whether it is deferred or known. // This lambda does the logic to propagate the third category. auto PropagateType = [&](Value *Source) -> DeducedType { // If the source argument is null, undef, or poison, then consider the // propagation to be untyped. This will fall through to the case where we // construct a nw if (doesNotImplyType(Source)) return nullptr; DeducedType SourceTy = computePointerElementType(Source); if (auto *Deferred = dyn_cast(SourceTy)) { LLVM_DEBUG(dbgs() << *Source << " will receive the same type as " << *V << "\n"); Deferred->Values.push_back(V); } return SourceTy; }; // These values have a natural pointer type (category 1). if (auto *GV = dyn_cast(V)) Ty = GV->getValueType(); else if (auto *Alloca = dyn_cast(V)) Ty = Alloca->getAllocatedType(); else if (auto *GEP = dyn_cast(V)) Ty = GEP->getResultElementType(); // These values have no intrinsic type (category 2). else if (isa(V) || isa(V)) Ty = nullptr; // These values propagate the source type (category 3). else if (auto *AS = dyn_cast(V)) Ty = PropagateType(AS->getPointerOperand()); else if (auto *Freeze = dyn_cast(V)) Ty = PropagateType(Freeze->getOperand(0)); // Yes, atomicrmw xchg can exchange a ptr type. This will also be considered // to propagate the source type. else if (auto *AI = dyn_cast(V)) Ty = PropagateType(AI->getValOperand()); // Selects and phis propagate types as well. Only investigate one of the // sources here as the type of the operation: if the primary operand has a // deferred type and a secondary operand has a known type, we'll discover that // when we handle uses anyways. else if (auto *Select = dyn_cast(V)) Ty = PropagateType(Select->getTrueValue()); else if (auto *Phi = dyn_cast(V)) Ty = PropagateType(Phi->getIncomingValue(0)); else if (auto *Arg = dyn_cast(V)) { // Check for an sret/byval/etc. attribute on the argument. If it doesn't // have one, then it will return null. There are other cases where we can // pre-fill the type of an argument, but that is handled in an earlier // pre-pass. unsigned ArgNo = Arg->getArgNo(); Ty = getParamType(Arg->getParent()->getAttributes(), ArgNo); } else if (auto *CB = dyn_cast(V)) { // TODO: Handle return types properly. Ty = Type::getInt8Ty(CB->getContext()); } else { // TODO: handle pointer-valued extractvalue, which probably comes from // cmpxchg or inlineasm. LLVM_DEBUG( dbgs() << "Value " << *V << " is not a known type of " << "pointer-valued instruction, this logic is probably wrong!\n"); } // If we haven't gotten a type at this point, we need to construct a new // deferred type to handle this value. This also considers cases where we // were trying to propagate a null constant. if (!Ty) { LLVM_DEBUG(dbgs() << "Value " << *V << " has no known type, creating a new type for it\n"); DeferredType *Deferred = new DeferredType; Deferred->Values.push_back(V); Ty = Deferred; } return Ty; } void SPIRVTypeScavenger::correctUseTypes(Instruction &I) { // This represents the types of all pointer-valued operands of the // instruction. SmallVector, 4> PointerOperands; // For instructions which operate with memory (e.g., load, store), this is the // value whose type will determine the type of the operand. When the memory // type is just a generic `ptr` type, this will be used to generate a // pointer-to-pointer-to-something type. auto GetMemoryType = [&](Value *V) -> DeducedType { if (V->getType()->isPointerTy()) return V; return V->getType(); }; // Basic instructions that have a clearly fixed type. if (auto *GEP = dyn_cast(&I)) { PointerOperands.emplace_back(GetElementPtrInst::getPointerOperandIndex(), GEP->getSourceElementType()); } else if (auto *LI = dyn_cast(&I)) { PointerOperands.emplace_back(LoadInst::getPointerOperandIndex(), GetMemoryType(LI)); } else if (auto *SI = dyn_cast(&I)) { PointerOperands.emplace_back(StoreInst::getPointerOperandIndex(), GetMemoryType(SI->getValueOperand())); } else if (auto *AI = dyn_cast(&I)) { PointerOperands.emplace_back(AtomicCmpXchgInst::getPointerOperandIndex(), GetMemoryType(AI->getCompareOperand())); } else if (auto *AI = dyn_cast(&I)) { PointerOperands.emplace_back(AtomicRMWInst::getPointerOperandIndex(), GetMemoryType(AI->getValOperand())); } else if (auto *CI = dyn_cast(&I)) { // icmp can compare pointers. If it isn't, ignore the instruction. if (!CI->getOperand(0)->getType()->isPointerTy()) return; // The two pointer operands should have the same type. PointerOperands.emplace_back(1, computePointerElementType(CI->getOperand(0))); } else if (auto *SI = dyn_cast(&I)) { if (!SI->getType()->isPointerTy()) return; // Both selected values should have the same type as the result. DeducedType Ty = computePointerElementType(SI); PointerOperands.emplace_back(1, Ty); PointerOperands.emplace_back(2, Ty); } else if (auto *Phi = dyn_cast(&I)) { if (!Phi->getType()->isPointerTy()) return; DeducedType Ty = computePointerElementType(Phi); for (Use &U : Phi->incoming_values()) { PointerOperands.emplace_back(U.getOperandNo(), Ty); } } else if (isa(&I) || isa(&I)) { if (!I.getType()->isPointerTy()) return; PointerOperands.emplace_back(0, computePointerElementType(&I)); } else if (auto *RI = dyn_cast(&I)) { if (!RI->getReturnValue() || !RI->getReturnValue()->getType()->isPointerTy()) return; // TODO: Handle return types properly. PointerOperands.emplace_back(0, Type::getInt8Ty(I.getContext())); } else if (auto *CB = dyn_cast(&I)) { PointerOperands.emplace_back(CB->getCalledOperandUse().getOperandNo(), CB->getFunctionType()); // If we have an identified function for the call instruction, map the // arguments we pass in to the argument requirements of the function. if (Function *F = CB->getCalledFunction()) { for (Use &U : CB->args()) { if (U->getType()->isPointerTy()) PointerOperands.emplace_back( U.getOperandNo(), computePointerElementType(F->getArg(CB->getArgOperandNo(&U)))); } } } // TODO: Handle insertvalue instructions that insert pointers. // Now that we've collected all the pointer-valued operands in the // instruction, go through and insert bitcasts for any operands that have the // wrong type, fix any deferred types whose types are now known, and merge any // deferred types that need to have the same type. IRBuilder Builder(&I); for (auto &Pair : PointerOperands) { Use &U = I.getOperandUse(Pair.first); DeducedType UsedTy = Pair.second; DeducedType SourceTy = computePointerElementType(U); // If we're handling a PHI node, we need to insert in the basic block that // the value comes in from, not immediately before this instruction. if (auto *Phi = dyn_cast(&I)) { BasicBlock *SourceBlock = Phi->getIncomingBlock(U); Builder.SetInsertPoint(SourceBlock->getTerminator()); } auto InsertCast = [&]() { if (isa(UsedTy)) { LLVM_DEBUG(dbgs() << "Inserting bitcast of " << *U.get() << " to change its type to " << *cast(UsedTy) << " because of use in " << *U.getUser() << "\n"); } else { LLVM_DEBUG(dbgs() << "Inserting bitcast of " << *U.get() << " for indirect pointer use of " << *cast(UsedTy) << " because of use in " << *U.getUser() << "\n"); } Value *CastedValue = Builder.Insert(CastInst::CreatePointerCast(U, U->getType())); DeducedTypes[CastedValue] = UsedTy; U.set(CastedValue); }; if (isa(UsedTy)) { // When the use is of an indirect-pointer type, insert a bitcast to the // use type only for this use. This prevents indirect pointers from // generally leaking into more of the type system and causing potential // issues. InsertCast(); } else if (auto *FixedTy = dyn_cast(SourceTy)) { if (auto *FixedUseTy = dyn_cast(UsedTy)) { // Both source and use type are fixed -> insert a bitcast are different. if (FixedTy != FixedUseTy) { InsertCast(); } } else if (auto *DeferredUseTy = dyn_cast(UsedTy)) { // Source type is fixed, use type is deferred: set the deferred type to // the fixed type. fixType(*DeferredUseTy, FixedTy); } } else if (auto *DeferredTy = dyn_cast(SourceTy)) { if (auto *FixedUseTy = dyn_cast(UsedTy)) { // Source type is fixed, use type is deferred: set the deferred type to // the fixed type. fixType(*DeferredTy, FixedUseTy); } else if (auto *DeferredUseTy = dyn_cast(UsedTy)) { // If they're both deferred, merge the two types together. mergeType(DeferredTy, DeferredUseTy); } } } } void SPIRVTypeScavenger::fixType(DeferredType &Ty, Type *ActualTy) { for (Value *V : Ty.Values) { LLVM_DEBUG(dbgs() << "Inferred type of " << *V << " to be " << *ActualTy << "\n"); DeducedTypes[V] = ActualTy; } delete &Ty; } void SPIRVTypeScavenger::mergeType(DeferredType *Ty1, DeferredType *Ty2) { // It's possible we're trying to merge the same type into itself. if (Ty1 == Ty2) return; for (Value *V : Ty2->Values) { DeducedTypes[V] = Ty1; Ty1->Values.push_back(V); } delete Ty2; } SPIRVTypeScavenger::PointeeType SPIRVTypeScavenger::getPointerElementType(Value *V) { PointerType *Ty = dyn_cast(V->getType()); assert(Ty && "Non-pointer types don't have pointee types"); if (!Ty->isOpaquePointerTy()) return Ty->getNonOpaquePointerElementType(); // Global values have a natural pointee type that we can use. if (auto *GV = dyn_cast(V)) return GV->getValueType(); // If we get a null/undef/poison value (this should be rare, but it can // happen if you use, e.g., store ptr null, ptr %val), then assume the result // should be an i8. This aligns with the use in the original deduction. if (doesNotImplyType(V)) { return Type::getInt8Ty(V->getContext()); } // If it's a constant expression, we won't have a type for it. Constant // expressions are currently translated via converting them to instructions // without a basic block. bool IsFromConstantExpr = isa(V) || (isa(V) && !cast(V)->getParent()); (void)IsFromConstantExpr; auto It = DeducedTypes.find(V); assert((It != DeducedTypes.end() || IsFromConstantExpr) && "How have we not typed the value?"); if (It != DeducedTypes.end()) { if (auto *Ty = dyn_cast(It->second)) return Ty; if (auto *ValTy = dyn_cast(It->second)) return ValTy; llvm_unreachable("Deferred types should have been resolved before now"); } return Type::getInt8Ty(V->getContext()); } Type *SPIRVTypeScavenger::getArgumentPointerElementType(Function *F, unsigned ArgNo) { return cast(getPointerElementType(F->getArg(ArgNo))); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVTypeScavenger.h000066400000000000000000000134241506646577400227440ustar00rootroot00000000000000//===- SPIRVTypeScavenger.h - Recover pointer types in opaque pointer IR --===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2022 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of The Khronos Group, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements the necessary logic to recover pointer types from LLVM // IR for the output SPIR-V file after LLVM IR completes its transition to // opaque pointers. // //===----------------------------------------------------------------------===// #ifndef SPIRVTYPESCAVENGER_H #define SPIRVTYPESCAVENGER_H #include "llvm/ADT/PointerUnion.h" #include "llvm/IR/Module.h" #include "llvm/IR/ValueMap.h" using namespace llvm; /// This class allows for the recovery of pointer element types from LLVM /// opaque pointer types. class SPIRVTypeScavenger { /// A representation of a pointer whose type will be determined by uses. This /// will include every value that needs to be assigned the same type. struct DeferredType { std::vector Values; }; /// This is called when a deferred type is fixed to a known use. void fixType(DeferredType &Deferred, Type *AssignedType); /// This merges two deferred types into one deferred type. void mergeType(DeferredType *A, DeferredType *B); /// A representation of the possible states of a type internal to this pass: /// it may be either /// * Something with a fixed LLVM type (this is a Type *) /// * A type whose pointer element type is yet unknown (DeferredType *) /// * A multi-level pointer type (this is a Value *, whose type is what this /// pointer type will point to). The latter should only exist for /// pointer operands of memory operations that return ptr. typedef PointerUnion DeducedType; friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, DeducedType Ty) { if (auto *AsTy = dyn_cast(Ty)) return OS << *AsTy; if (auto *AsDeferred = dyn_cast(Ty)) return OS << "deferred type for " << *AsDeferred->Values[0]; if (auto *AsValue = dyn_cast(Ty)) return OS << "points to " << *AsValue; return OS; } /// Compute the pointer element type of a value, solely based on its /// definition. The value must be pointer-valued. DeducedType computePointerElementType(Value *V); /// This stores the Value -> pointer element type mapping for the module. ValueMap DeducedTypes; /// Enforce that the pointer element types of all operands of the instruction /// matches the type that the instruction itself requires. If a pointer /// element type of one of the operands is deferred, this will type the use /// correctly. void correctUseTypes(Instruction &I); /// This assigns known pointer element types for the parameters of a function. /// This method should be called for all functions before doing any type /// analysis on the module. void deduceFunctionType(Function &F); /// This assigns known pointer element types for parameters of LLVM /// intrinsics. void deduceIntrinsicTypes(Function &F, Intrinsic::ID Id); /// Compute pointer element types for all pertinent values in the module. void typeModule(Module &M); public: explicit SPIRVTypeScavenger(Module &M) { typeModule(M); } /// This type represents the type that a pointer element type of a type. If it /// is a Type value, then the pointee type represents a pointer to that type. /// If it is a Value value, then the pointee type is the type of that value /// (which should be a pointer-typed value.) typedef PointerUnion PointeeType; /// Get the pointer element type of the value. /// If the type is a multi-level pointer, then PointeeType will be a Value /// whose pointee type can be recursively queried through this method. /// Otherwise, it will be a pointer to the Type returned by this method. PointeeType getPointerElementType(Value *V); /// Get the pointer element type of an argument of the given function. Since /// this type is guaranteed to not be a multi-level pointer type, the result /// is an LLVM type instead of a PointeeType. Type *getArgumentPointerElementType(Function *F, unsigned ArgNo); }; #endif // SPIRVTYPESCAVENGER_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVUtil.cpp000066400000000000000000002741551506646577400214470ustar00rootroot00000000000000//===- SPIRVUtil.cpp - SPIR-V Utilities -------------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines utility classes and functions shared by SPIR-V /// reader/writer. /// //===----------------------------------------------------------------------===// #include "FunctionDescriptor.h" #include "ManglingUtils.h" #include "NameMangleAPI.h" #include "OCLUtil.h" #include "ParameterType.h" #include "SPIRVInternal.h" #include "SPIRVMDWalker.h" #include "libSPIRV/SPIRVDecorate.h" #include "libSPIRV/SPIRVValue.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Demangle/Demangle.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ToolOutputFile.h" #include #include #define DEBUG_TYPE "spirv" namespace { // Return the value for when the dimension index of a builtin is out of range. uint64_t getBuiltinOutOfRangeValue(StringRef VarName) { assert(VarName.startswith("__spirv_BuiltIn")); return StringSwitch(VarName) .EndsWith("GlobalSize", 1) .EndsWith("NumWorkgroups", 1) .EndsWith("WorkgroupSize", 1) .EndsWith("EnqueuedWorkgroupSize", 1) .Default(0); } } // anonymous namespace namespace SPIRV { #ifdef _SPIRV_SUPPORT_TEXT_FMT cl::opt UseTextFormat("spirv-text", cl::desc("Use text format for SPIR-V for debugging purpose"), cl::location(SPIRVUseTextFormat)); #endif #ifdef _SPIRVDBG cl::opt EnableDbgOutput("spirv-debug", cl::desc("Enable SPIR-V debug output"), cl::location(SPIRVDbgEnable)); #endif bool isSupportedTriple(Triple T) { return T.isSPIR() || T.isSPIRV(); } void addFnAttr(CallInst *Call, Attribute::AttrKind Attr) { Call->addFnAttr(Attr); } void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr) { Call->removeFnAttr(Attr); } Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder) { unsigned OldSize = cast(V->getType())->getNumElements(); unsigned NewSize = NewType->getNumElements(); assert(OldSize < NewSize); std::vector Components; IntegerType *Int32Ty = Builder.getInt32Ty(); for (unsigned I = 0; I < NewSize; I++) { if (I < OldSize) Components.push_back(ConstantInt::get(Int32Ty, I)); else Components.push_back(PoisonValue::get(Int32Ty)); } return Builder.CreateShuffleVector(V, PoisonValue::get(V->getType()), ConstantVector::get(Components), "vecext"); } void saveLLVMModule(Module *M, const std::string &OutputFile) { std::error_code EC; ToolOutputFile Out(OutputFile.c_str(), EC, sys::fs::OF_None); if (EC) { SPIRVDBG(errs() << "Fails to open output file: " << EC.message();) return; } WriteBitcodeToFile(*M, Out.os()); Out.keep(); } std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed, Type *PET) { if (Ty->isHalfTy()) return "half"; if (Ty->isFloatTy()) return "float"; if (Ty->isDoubleTy()) return "double"; if (auto IntTy = dyn_cast(Ty)) { std::string SignPrefix; std::string Stem; if (!Signed) SignPrefix = "u"; switch (IntTy->getIntegerBitWidth()) { case 8: Stem = "char"; break; case 16: Stem = "short"; break; case 32: Stem = "int"; break; case 64: Stem = "long"; break; default: Stem = "invalid_type"; break; } return SignPrefix + Stem; } if (auto VecTy = dyn_cast(Ty)) { Type *EleTy = VecTy->getElementType(); unsigned Size = VecTy->getNumElements(); std::stringstream Ss; Ss << mapLLVMTypeToOCLType(EleTy, Signed) << Size; return Ss.str(); } // It is expected that `Ty` can be mapped to `ReturnType` from "Optional // Postfixes for SPIR-V Builtin Function Names" section of // SPIRVRepresentationInLLVM.rst document (aka SPIRV-friendly IR). // If `Ty` is not a scalar or vector type mentioned in the document (return // value of some SPIR-V instructions may be represented as pointer to a struct // in LLVM IR) we can mangle the type. BuiltinFuncMangleInfo MangleInfo; if (Ty->isPointerTy()) { assert(cast(const_cast(Ty)) ->isOpaqueOrPointeeTypeMatches(PET)); MangleInfo.getTypeMangleInfo(0).PointerElementType.setPointer(PET); } std::string MangledName = mangleBuiltin("", const_cast(Ty), &MangleInfo); // Remove "_Z0"(3 characters) from the front of the name return MangledName.erase(0, 3); } std::string mapSPIRVTypeToOCLType(SPIRVType *Ty, bool Signed) { if (Ty->isTypeFloat()) { auto W = Ty->getBitWidth(); switch (W) { case 16: return "half"; case 32: return "float"; case 64: return "double"; default: assert(0 && "Invalid floating pointer type"); return std::string("float") + W + "_t"; } } if (Ty->isTypeInt()) { std::string SignPrefix; std::string Stem; if (!Signed) SignPrefix = "u"; auto W = Ty->getBitWidth(); switch (W) { case 8: Stem = "char"; break; case 16: Stem = "short"; break; case 32: Stem = "int"; break; case 64: Stem = "long"; break; default: llvm_unreachable("Invalid integer type"); Stem = std::string("int") + W + "_t"; break; } return SignPrefix + Stem; } if (Ty->isTypeVector()) { auto EleTy = Ty->getVectorComponentType(); auto Size = Ty->getVectorComponentCount(); std::stringstream Ss; Ss << mapSPIRVTypeToOCLType(EleTy, Signed) << Size; return Ss.str(); } llvm_unreachable("Invalid type"); return "unknown_type"; } PointerType *getOrCreateOpaquePtrType(Module *M, const std::string &Name, unsigned AddrSpace) { return PointerType::get(getOrCreateOpaqueStructType(M, Name), AddrSpace); } StructType *getOrCreateOpaqueStructType(Module *M, StringRef Name) { auto OpaqueType = StructType::getTypeByName(M->getContext(), Name); if (!OpaqueType) OpaqueType = StructType::create(M->getContext(), Name); return OpaqueType; } PointerType *getSamplerType(Module *M) { return getOrCreateOpaquePtrType(M, getSPIRVTypeName(kSPIRVTypeName::Sampler), SPIRAS_Constant); } Type *getSamplerStructType(Module *M) { return getOrCreateOpaqueStructType(M, getSPIRVTypeName(kSPIRVTypeName::Sampler)); } PointerType *getSPIRVOpaquePtrType(Module *M, Op OC) { std::string Name = getSPIRVTypeName(SPIRVOpaqueTypeOpCodeMap::rmap(OC)); return getOrCreateOpaquePtrType(M, Name, getOCLOpaqueTypeAddrSpace(OC)); } void getFunctionTypeParameterTypes(llvm::FunctionType *FT, SmallVector &ArgTys) { for (auto I = FT->param_begin(), E = FT->param_end(); I != E; ++I) { ArgTys.push_back(*I); } } bool isVoidFuncTy(FunctionType *FT) { return FT->getReturnType()->isVoidTy(); } bool isOCLImageStructType(llvm::Type *Ty, StringRef *Name) { if (auto *ST = dyn_cast_or_null(Ty)) if (ST->isOpaque()) { auto FullName = ST->getName(); if (FullName.find(kSPR2TypeName::ImagePrefix) == 0) { if (Name) *Name = FullName.drop_front(strlen(kSPR2TypeName::OCLPrefix)); return true; } } return false; } /// \param BaseTyName is the type Name as in spirv.BaseTyName.Postfixes /// \param Postfix contains postfixes extracted from the SPIR-V image /// type Name as spirv.BaseTyName.Postfixes. bool isSPIRVStructType(llvm::Type *Ty, StringRef BaseTyName, StringRef *Postfix) { auto *ST = dyn_cast(Ty); if (!ST) return false; if (ST->isOpaque()) { auto FullName = ST->getName(); std::string N = std::string(kSPIRVTypeName::PrefixAndDelim) + BaseTyName.str(); if (FullName != N) N = N + kSPIRVTypeName::Delimiter; if (FullName.startswith(N)) { if (Postfix) *Postfix = FullName.drop_front(N.size()); return true; } } return false; } bool isSYCLHalfType(llvm::Type *Ty) { if (auto *ST = dyn_cast(Ty)) { if (!ST->hasName()) return false; StringRef Name = ST->getName(); Name.consume_front("class."); if ((Name.startswith("cl::sycl::") || Name.startswith("__sycl_internal::")) && Name.endswith("::half")) { return true; } } return false; } bool isSYCLBfloat16Type(llvm::Type *Ty) { if (auto *ST = dyn_cast(Ty)) { if (!ST->hasName()) return false; StringRef Name = ST->getName(); Name.consume_front("class."); if ((Name.startswith("cl::sycl::") || Name.startswith("__sycl_internal::")) && Name.endswith("::bfloat16")) { return true; } } return false; } Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef ArgTypes, StringRef Name, BuiltinFuncMangleInfo *Mangle, AttributeList *Attrs, bool TakeName) { std::string MangledName{Name}; bool IsVarArg = false; if (Mangle) { MangledName = mangleBuiltin(Name, ArgTypes, Mangle); IsVarArg = 0 <= Mangle->getVarArg(); if (IsVarArg) ArgTypes = ArgTypes.slice(0, Mangle->getVarArg()); } FunctionType *FT = FunctionType::get(RetTy, ArgTypes, IsVarArg); Function *F = M->getFunction(MangledName); if (!TakeName && F && F->getFunctionType() != FT && Mangle != nullptr) { std::string S; raw_string_ostream SS(S); SS << "Error: Attempt to redefine function: " << *F << " => " << *FT << '\n'; report_fatal_error(llvm::Twine(SS.str()), false); } if (!F || F->getFunctionType() != FT) { auto NewF = Function::Create(FT, GlobalValue::ExternalLinkage, MangledName, M); if (F && TakeName) { NewF->takeName(F); LLVM_DEBUG( dbgs() << "[getOrCreateFunction] Warning: taking function Name\n"); } if (NewF->getName() != MangledName) { LLVM_DEBUG( dbgs() << "[getOrCreateFunction] Warning: function Name changed\n"); } LLVM_DEBUG(dbgs() << "[getOrCreateFunction] "; if (F) dbgs() << *F << " => "; dbgs() << *NewF << '\n';); if (F) NewF->setDSOLocal(F->isDSOLocal()); F = NewF; F->setCallingConv(CallingConv::SPIR_FUNC); if (Attrs) F->setAttributes(*Attrs); } return F; } std::vector getArguments(CallInst *CI, unsigned Start, unsigned End) { std::vector Args; if (End == 0) End = CI->arg_size(); for (; Start != End; ++Start) { Args.push_back(CI->getArgOperand(Start)); } return Args; } uint64_t getArgAsInt(CallInst *CI, unsigned I) { return cast(CI->getArgOperand(I))->getZExtValue(); } Scope getArgAsScope(CallInst *CI, unsigned I) { return static_cast(getArgAsInt(CI, I)); } Decoration getArgAsDecoration(CallInst *CI, unsigned I) { return static_cast(getArgAsInt(CI, I)); } std::string decorateSPIRVFunction(const std::string &S) { return std::string(kSPIRVName::Prefix) + S + kSPIRVName::Postfix; } StringRef undecorateSPIRVFunction(StringRef S) { assert(S.find(kSPIRVName::Prefix) == 0); const size_t Start = strlen(kSPIRVName::Prefix); auto End = S.rfind(kSPIRVName::Postfix); return S.substr(Start, End - Start); } std::string prefixSPIRVName(const std::string &S) { return std::string(kSPIRVName::Prefix) + S; } StringRef dePrefixSPIRVName(StringRef R, SmallVectorImpl &Postfix) { const size_t Start = strlen(kSPIRVName::Prefix); if (!R.startswith(kSPIRVName::Prefix)) return R; R = R.drop_front(Start); R.split(Postfix, "_", -1, false); auto Name = Postfix.front(); Postfix.erase(Postfix.begin()); return Name; } std::string getSPIRVFuncName(Op OC, StringRef PostFix) { return prefixSPIRVName(getName(OC) + PostFix.str()); } std::string getSPIRVFuncName(Op OC, const Type *PRetTy, bool IsSigned, Type *PET) { return prefixSPIRVName(getName(OC) + kSPIRVPostfix::Divider + getPostfixForReturnType(PRetTy, IsSigned, PET)); } std::string getSPIRVFuncName(SPIRVBuiltinVariableKind BVKind) { return prefixSPIRVName(getName(BVKind)); } std::string getSPIRVExtFuncName(SPIRVExtInstSetKind Set, unsigned ExtOp, StringRef PostFix) { std::string ExtOpName; switch (Set) { default: llvm_unreachable("invalid extended instruction set"); ExtOpName = "unknown"; break; case SPIRVEIS_OpenCL: ExtOpName = getName(static_cast(ExtOp)); break; } return prefixSPIRVName(SPIRVExtSetShortNameMap::map(Set) + '_' + ExtOpName + PostFix.str()); } SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target) { if (Postfix == kSPIRVPostfix::Sat) return new SPIRVDecorate(spv::DecorationSaturatedConversion, Target); if (Postfix.startswith(kSPIRVPostfix::Rt)) return new SPIRVDecorate(spv::DecorationFPRoundingMode, Target, map(Postfix.str())); return nullptr; } SPIRVValue *addDecorations(SPIRVValue *Target, const SmallVectorImpl &Decs) { for (auto &I : Decs) if (auto Dec = mapPostfixToDecorate(I, Target)) Target->addDecorate(Dec); return Target; } std::string getPostfix(Decoration Dec, unsigned Value) { switch (Dec) { default: llvm_unreachable("not implemented"); return "unknown"; case spv::DecorationSaturatedConversion: return kSPIRVPostfix::Sat; case spv::DecorationFPRoundingMode: return rmap(static_cast(Value)); } } std::string getPostfixForReturnType(CallInst *CI, bool IsSigned) { return getPostfixForReturnType(CI->getType(), IsSigned); } std::string getPostfixForReturnType(const Type *PRetTy, bool IsSigned, Type *PET) { return std::string(kSPIRVPostfix::Return) + mapLLVMTypeToOCLType(PRetTy, IsSigned, PET); } // Enqueue kernel, kernel query, pipe and address space cast built-ins // are not mangled. bool isNonMangledOCLBuiltin(StringRef Name) { if (!Name.startswith("__")) return false; return isEnqueueKernelBI(Name) || isKernelQueryBI(Name) || isPipeOrAddressSpaceCastBI(Name.drop_front(2)); } Op getSPIRVFuncOC(StringRef S, SmallVectorImpl *Dec) { Op OC; SmallVector Postfix; StringRef Name; if (!oclIsBuiltin(S, Name)) Name = S; StringRef R(Name); if ((!Name.startswith(kSPIRVName::Prefix) && !isNonMangledOCLBuiltin(S)) || !getByName(dePrefixSPIRVName(R, Postfix).str(), OC)) { return OpNop; } if (Dec) for (auto &I : Postfix) Dec->push_back(I.str()); return OC; } bool getSPIRVBuiltin(const std::string &OrigName, spv::BuiltIn &B) { SmallVector Postfix; StringRef R(OrigName); R = dePrefixSPIRVName(R, Postfix); if (!Postfix.empty()) return false; return getByName(R.str(), B); } // Demangled name is a substring of the name. The DemangledName is updated only // if true is returned bool oclIsBuiltin(StringRef Name, StringRef &DemangledName, bool IsCpp) { if (Name == "printf") { DemangledName = Name; return true; } if (isNonMangledOCLBuiltin(Name)) { DemangledName = Name.drop_front(2); return true; } if (!Name.startswith("_Z")) return false; // OpenCL C++ built-ins are declared in cl namespace. // TODO: consider using 'St' abbriviation for cl namespace mangling. // Similar to ::std:: in C++. if (IsCpp) { if (!Name.startswith("_ZN")) return false; // Skip CV and ref qualifiers. size_t NameSpaceStart = Name.find_first_not_of("rVKRO", 3); // All built-ins are in the ::cl:: namespace. if (Name.substr(NameSpaceStart, 11) != "2cl7__spirv") return false; size_t DemangledNameLenStart = NameSpaceStart + 11; size_t Start = Name.find_first_not_of("0123456789", DemangledNameLenStart); size_t Len = 0; Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart) .getAsInteger(10, Len); DemangledName = Name.substr(Start, Len); } else { size_t Start = Name.find_first_not_of("0123456789", 2); size_t Len = 0; Name.substr(2, Start - 2).getAsInteger(10, Len); DemangledName = Name.substr(Start, Len); } return true; } // Check if a mangled type Name is unsigned bool isMangledTypeUnsigned(char Mangled) { return Mangled == 'h' /* uchar */ || Mangled == 't' /* ushort */ || Mangled == 'j' /* uint */ || Mangled == 'm' /* ulong */; } // Check if a mangled type Name is signed bool isMangledTypeSigned(char Mangled) { return Mangled == 'c' /* char */ || Mangled == 'a' /* signed char */ || Mangled == 's' /* short */ || Mangled == 'i' /* int */ || Mangled == 'l' /* long */; } // Check if a mangled type Name is floating point (excludes half) bool isMangledTypeFP(char Mangled) { return Mangled == 'f' /* float */ || Mangled == 'd'; /* double */ } // Check if a mangled type Name is half bool isMangledTypeHalf(std::string Mangled) { return Mangled == "Dh"; /* half */ } void eraseSubstitutionFromMangledName(std::string &MangledName) { auto Len = MangledName.length(); while (Len >= 2 && MangledName.substr(Len - 2, 2) == "S_") { Len -= 2; MangledName.erase(Len, 2); } } ParamType lastFuncParamType(StringRef MangledName) { std::string Copy(MangledName); eraseSubstitutionFromMangledName(Copy); char Mangled = Copy.back(); std::string Mangled2 = Copy.substr(Copy.size() - 2); std::string Mangled5 = Copy.substr(Copy.size() - 5); if (Mangled5 == "DF16b") { return ParamType::FLOAT; } if (isMangledTypeFP(Mangled) || isMangledTypeHalf(Mangled2)) { return ParamType::FLOAT; } else if (isMangledTypeUnsigned(Mangled)) { return ParamType::UNSIGNED; } else if (isMangledTypeSigned(Mangled)) { return ParamType::SIGNED; } return ParamType::UNKNOWN; } // Check if the last argument is signed bool isLastFuncParamSigned(StringRef MangledName) { return lastFuncParamType(MangledName) == ParamType::SIGNED; } // Check if a mangled function Name contains unsigned atomic type bool containsUnsignedAtomicType(StringRef Name) { auto Loc = Name.find(kMangledName::AtomicPrefixIncoming); if (Loc == StringRef::npos) return false; return isMangledTypeUnsigned( Name[Loc + strlen(kMangledName::AtomicPrefixIncoming)]); } Constant *castToVoidFuncPtr(Function *F) { auto T = getVoidFuncPtrType(F->getParent()); return ConstantExpr::getBitCast(F, T); } bool hasArrayArg(Function *F) { for (auto I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { LLVM_DEBUG(dbgs() << "[hasArrayArg] " << *I << '\n'); if (I->getType()->isArrayTy()) { return true; } } return false; } /// Convert a struct name from the name given to it in Itanium name mangling to /// the name given to it as an LLVM opaque struct. static std::string demangleBuiltinOpenCLTypeName(StringRef MangledStructName) { assert(MangledStructName.startswith("ocl_") && "Not a valid builtin OpenCL mangled name"); // Bare structure type that starts with ocl_ is a builtin opencl type. // See clang/lib/CodeGen/CGOpenCLRuntime for how these map to LLVM types // and clang/lib/AST/ItaniumMangle for how they are mangled. // In general, ocl_ is mapped to pointer-to-%opencl., but // there is some variance around whether or not _t is included in the // mangled name. std::string LlvmStructName = StringSwitch(MangledStructName) .Case("ocl_sampler", "opencl.sampler_t") .Case("ocl_event", "opencl.event_t") .Case("ocl_clkevent", "opencl.clk_event_t") .Case("ocl_queue", "opencl.queue_t") .Case("ocl_reserveid", "opencl.reserve_id_t") .Default("") .str(); if (LlvmStructName.empty()) { LlvmStructName = "opencl."; LlvmStructName += MangledStructName.substr(4); // Strip off ocl_ if (!MangledStructName.endswith("_t")) LlvmStructName += "_t"; } return LlvmStructName; } void getParameterTypes(Function *F, SmallVectorImpl &ArgTys) { // If there's no mangled name, we can't do anything. Also, if there's no // parameters, do nothing. StringRef Name = F->getName(); if (!Name.startswith("_Z") || F->arg_empty()) return; // Start by filling in a skeleton of information we can get from the LLVM type // itself. ArgTys.clear(); auto *FT = F->getFunctionType(); ArgTys.reserve(FT->getNumParams()); bool HasSret = false; for (Argument &Arg : F->args()) { if (!Arg.getType()->isPointerTy()) ArgTys.push_back(nullptr); else if (Type *Ty = Arg.getParamStructRetType()) { assert(!HasSret && &Arg == F->getArg(0) && "sret parameter should only appear on the first argument"); HasSret = true; ArgTys.push_back(dyn_cast(Ty)); } else { ArgTys.push_back(nullptr); } } Module *M = F->getParent(); // Skip the first argument if it's an sret parameter--this would be an // implicit parameter not recognized as part of the function parameters. auto *ArgIter = ArgTys.begin(); if (HasSret) ++ArgIter; // Demangle the function arguments. If we get an input name of // "_Z12write_imagei20ocl_image1d_array_woDv2_iiDv4_i", then we expect // that Demangler.getFunctionParameters will return // "(ocl_image1d_array_wo, int __vector(2), int, int __vector(4))" (in other // words, the stuff between the parentheses if you ran C++ filt, including // the parentheses itself). ItaniumPartialDemangler Demangler; std::string MangledName = F->getName().str(); if (Demangler.partialDemangle(MangledName.c_str())) return; char *Buf = nullptr; size_t BufLen = 0; Buf = Demangler.getFunctionParameters(Buf, &BufLen); StringRef Args(Buf, BufLen); // Strip parentheses from the result. Args = Args.slice(1, Args.size() - 2); if (Args.find_first_of("<(") == StringRef::npos) { // Go through the function arguments using type names where possible. SmallVector ArgParams; Args.split(ArgParams, ", "); // Sanity check that the name mangling matches up to the expected number of // arguments. if (ArgParams.size() > (size_t)(ArgTys.end() - ArgIter)) { LLVM_DEBUG(dbgs() << "[getParameterTypes] function " << F->getName() << " appears to have " << ArgParams.size() << " arguments but has " << (ArgTys.end() - ArgIter) << "\n"); free(Buf); return; } for (StringRef Arg : ArgParams) { StructType *Pointee = nullptr; if (Arg.endswith("*") && !Arg.endswith("**")) { // Strip off address space and other qualifiers. StringRef MangledStructName = Arg.split(' ').first; if (MangledStructName.consume_front("__spirv_")) { // This is a pointer to a SPIR-V OpType* opaque struct. In general, // convert __spirv_[__Suffix] to %spirv.Type[._Suffix] auto NameSuffixPair = MangledStructName.split('_'); std::string StructName = "spirv."; StructName += NameSuffixPair.first; if (!NameSuffixPair.second.empty()) { StructName += "."; StructName += NameSuffixPair.second; } Pointee = getOrCreateOpaqueStructType(M, StructName); } else if (MangledStructName.startswith("opencl.")) { Pointee = getOrCreateOpaqueStructType(M, MangledStructName); } } else if (!Arg.contains(' ') && Arg.startswith("ocl_")) { std::string StructName = demangleBuiltinOpenCLTypeName(Arg); Pointee = getOrCreateOpaqueStructType(M, StructName); } *ArgIter++ = Pointee; } } free(Buf); } // This is a transitional helper function to fill in mangling information for // mangleBuiltin while all the calls to mutateCallInst are being transitioned. static void typeMangle(BuiltinFuncMangleInfo *Mangle, ArrayRef Args) { if (!Mangle) return; for (unsigned I = 0; I < Args.size(); I++) if (Args[I]->getType()->isPointerTy()) { auto &PointeeTy = Mangle->getTypeMangleInfo(I).PointerElementType; PointeeTy.setPointer( Args[I]->getType()->getNonOpaquePointerElementType()); if (PointeeTy.getPointer()->isPointerTy()) { PointeeTy.setPointer( PointeeTy.getPointer()->getNonOpaquePointerElementType()); PointeeTy.setInt(true); } } } CallInst *mutateCallInst( Module *M, CallInst *CI, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle, AttributeList *Attrs, bool TakeFuncName) { LLVM_DEBUG(dbgs() << "[mutateCallInst] " << *CI); auto Args = getArguments(CI); auto NewName = ArgMutate(CI, Args); std::string InstName; if (!CI->getType()->isVoidTy() && CI->hasName()) { InstName = CI->getName().str(); CI->setName(InstName + ".old"); } typeMangle(Mangle, Args); auto NewCI = addCallInst(M, NewName, CI->getType(), Args, Attrs, CI, Mangle, InstName, TakeFuncName); NewCI->setDebugLoc(CI->getDebugLoc()); NewCI->copyMetadata(*CI); NewCI->setAttributes(CI->getAttributes()); NewCI->setTailCall(CI->isTailCall()); if (isa(CI)) NewCI->setFastMathFlags(CI->getFastMathFlags()); if (CI->hasFnAttr("fpbuiltin-max-error")) { auto Attr = CI->getFnAttr("fpbuiltin-max-error"); NewCI->addFnAttr(Attr); } LLVM_DEBUG(dbgs() << " => " << *NewCI << '\n'); CI->replaceAllUsesWith(NewCI); CI->eraseFromParent(); return NewCI; } Instruction *mutateCallInst( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle, AttributeList *Attrs, bool TakeFuncName) { LLVM_DEBUG(dbgs() << "[mutateCallInst] " << *CI); auto Args = getArguments(CI); Type *RetTy = CI->getType(); auto NewName = ArgMutate(CI, Args, RetTy); StringRef InstName = CI->getName(); typeMangle(Mangle, Args); auto NewCI = addCallInst(M, NewName, RetTy, Args, Attrs, CI, Mangle, InstName, TakeFuncName); auto NewI = RetMutate(NewCI); NewI->takeName(CI); NewI->setDebugLoc(CI->getDebugLoc()); LLVM_DEBUG(dbgs() << " => " << *NewI << '\n'); if (!CI->getType()->isVoidTy()) CI->replaceAllUsesWith(NewI); CI->eraseFromParent(); return NewI; } void mutateFunction( Function *F, std::function &)> ArgMutate, BuiltinFuncMangleInfo *Mangle, AttributeList *Attrs, bool TakeFuncName) { auto M = F->getParent(); for (auto I = F->user_begin(), E = F->user_end(); I != E;) { if (auto CI = dyn_cast(*I++)) mutateCallInst(M, CI, ArgMutate, Mangle, Attrs, TakeFuncName); } if (F->use_empty()) F->eraseFromParent(); } void mutateFunction( Function *F, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, BuiltinFuncMangleInfo *Mangle, AttributeList *Attrs, bool TakeName) { auto *M = F->getParent(); for (auto I = F->user_begin(), E = F->user_end(); I != E;) { if (auto *CI = dyn_cast(*I++)) mutateCallInst(M, CI, ArgMutate, RetMutate, Mangle, Attrs, TakeName); } if (F->use_empty()) F->eraseFromParent(); } CallInst *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &)> ArgMutate, AttributeList *Attrs) { BuiltinFuncMangleInfo BtnInfo; return mutateCallInst(M, CI, ArgMutate, &BtnInfo, Attrs); } Instruction *mutateCallInstSPIRV( Module *M, CallInst *CI, std::function &, Type *&RetTy)> ArgMutate, std::function RetMutate, AttributeList *Attrs) { BuiltinFuncMangleInfo BtnInfo; return mutateCallInst(M, CI, ArgMutate, RetMutate, &BtnInfo, Attrs); } CallInst *addCallInst(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, Instruction *Pos, BuiltinFuncMangleInfo *Mangle, StringRef InstName, bool TakeFuncName) { auto F = getOrCreateFunction(M, RetTy, getTypes(Args), FuncName, Mangle, Attrs, TakeFuncName); // Cannot assign a Name to void typed values auto CI = CallInst::Create(F, Args, RetTy->isVoidTy() ? "" : InstName, Pos); CI->setCallingConv(F->getCallingConv()); CI->setAttributes(F->getAttributes()); return CI; } CallInst *addCallInstSPIRV(Module *M, StringRef FuncName, Type *RetTy, ArrayRef Args, AttributeList *Attrs, ArrayRef PointerElementTypes, Instruction *Pos, StringRef InstName) { BuiltinFuncMangleInfo BtnInfo; for (unsigned I = 0; I < PointerElementTypes.size(); I++) { BtnInfo.getTypeMangleInfo(I).PointerElementType.setPointer( PointerElementTypes[I]); if (Args[I]->getType()->isPointerTy()) assert(cast(Args[I]->getType()) ->isOpaqueOrPointeeTypeMatches(PointerElementTypes[I])); } return addCallInst(M, FuncName, RetTy, Args, Attrs, Pos, &BtnInfo, InstName); } bool isValidVectorSize(unsigned I) { return I == 2 || I == 3 || I == 4 || I == 8 || I == 16; } Value *addVector(Instruction *InsPos, ValueVecRange Range) { size_t VecSize = Range.second - Range.first; if (VecSize == 1) return *Range.first; assert(isValidVectorSize(VecSize) && "Invalid vector size"); IRBuilder<> Builder(InsPos); auto Vec = Builder.CreateVectorSplat(VecSize, *Range.first); unsigned Index = 1; for (++Range.first; Range.first != Range.second; ++Range.first, ++Index) Vec = Builder.CreateInsertElement( Vec, *Range.first, ConstantInt::get(Type::getInt32Ty(InsPos->getContext()), Index, false)); return Vec; } void makeVector(Instruction *InsPos, std::vector &Ops, ValueVecRange Range) { auto Vec = addVector(InsPos, Range); Ops.erase(Range.first, Range.second); Ops.push_back(Vec); } void expandVector(Instruction *InsPos, std::vector &Ops, size_t VecPos) { auto Vec = Ops[VecPos]; auto *VT = dyn_cast(Vec->getType()); if (!VT) return; size_t N = VT->getNumElements(); IRBuilder<> Builder(InsPos); for (size_t I = 0; I != N; ++I) Ops.insert(Ops.begin() + VecPos + I, Builder.CreateExtractElement( Vec, ConstantInt::get(Type::getInt32Ty(InsPos->getContext()), I, false))); Ops.erase(Ops.begin() + VecPos + N); } Constant *castToInt8Ptr(Constant *V, unsigned Addr = 0) { return ConstantExpr::getBitCast(V, Type::getInt8PtrTy(V->getContext(), Addr)); } PointerType *getInt8PtrTy(PointerType *T) { return Type::getInt8PtrTy(T->getContext(), T->getAddressSpace()); } Value *castToInt8Ptr(Value *V, Instruction *Pos) { return CastInst::CreatePointerCast( V, getInt8PtrTy(cast(V->getType())), "", Pos); } CallInst *addBlockBind(Module *M, Function *InvokeFunc, Value *BlkCtx, Value *CtxLen, Value *CtxAlign, Instruction *InsPos, StringRef InstName) { auto BlkTy = getOrCreateOpaquePtrType(M, SPIR_TYPE_NAME_BLOCK_T, SPIRAS_Private); auto &Ctx = M->getContext(); Value *BlkArgs[] = { castToInt8Ptr(InvokeFunc), CtxLen ? CtxLen : UndefValue::get(Type::getInt32Ty(Ctx)), CtxAlign ? CtxAlign : UndefValue::get(Type::getInt32Ty(Ctx)), BlkCtx ? BlkCtx : UndefValue::get(Type::getInt8PtrTy(Ctx))}; return addCallInst(M, SPIR_INTRINSIC_BLOCK_BIND, BlkTy, BlkArgs, nullptr, InsPos, nullptr, InstName); } IntegerType *getSizetType(Module *M) { return IntegerType::getIntNTy(M->getContext(), M->getDataLayout().getPointerSizeInBits(0)); } Type *getVoidFuncType(Module *M) { return FunctionType::get(Type::getVoidTy(M->getContext()), false); } Type *getVoidFuncPtrType(Module *M, unsigned AddrSpace) { return PointerType::get(getVoidFuncType(M), AddrSpace); } ConstantInt *getInt64(Module *M, int64_t Value) { return ConstantInt::getSigned(Type::getInt64Ty(M->getContext()), Value); } ConstantInt *getUInt64(Module *M, uint64_t Value) { return ConstantInt::get(Type::getInt64Ty(M->getContext()), Value, false); } Constant *getFloat32(Module *M, float Value) { return ConstantFP::get(Type::getFloatTy(M->getContext()), Value); } ConstantInt *getInt32(Module *M, int Value) { return ConstantInt::get(Type::getInt32Ty(M->getContext()), Value, true); } ConstantInt *getUInt32(Module *M, unsigned Value) { return ConstantInt::get(Type::getInt32Ty(M->getContext()), Value, false); } ConstantInt *getInt(Module *M, int64_t Value) { return Value >> 32 ? getInt64(M, Value) : getInt32(M, static_cast(Value)); } ConstantInt *getUInt(Module *M, uint64_t Value) { return Value >> 32 ? getUInt64(M, Value) : getUInt32(M, static_cast(Value)); } ConstantInt *getUInt16(Module *M, unsigned short Value) { return ConstantInt::get(Type::getInt16Ty(M->getContext()), Value, false); } std::vector getInt32(Module *M, const std::vector &Values) { std::vector V; for (auto &I : Values) V.push_back(getInt32(M, I)); return V; } ConstantInt *getSizet(Module *M, uint64_t Value) { return ConstantInt::get(getSizetType(M), Value, false); } /////////////////////////////////////////////////////////////////////////////// // // Functions for getting metadata // /////////////////////////////////////////////////////////////////////////////// int64_t getMDOperandAsInt(MDNode *N, unsigned I) { return mdconst::dyn_extract(N->getOperand(I))->getZExtValue(); } // Additional helper function to be reused by getMDOperandAs* helpers Metadata *getMDOperandOrNull(MDNode *N, unsigned I) { if (!N) return nullptr; return N->getOperand(I); } StringRef getMDOperandAsString(MDNode *N, unsigned I) { if (auto *Str = dyn_cast_or_null(getMDOperandOrNull(N, I))) return Str->getString(); return ""; } MDNode *getMDOperandAsMDNode(MDNode *N, unsigned I) { return dyn_cast_or_null(getMDOperandOrNull(N, I)); } Type *getMDOperandAsType(MDNode *N, unsigned I) { return cast(N->getOperand(I))->getType(); } std::set getNamedMDAsStringSet(Module *M, const std::string &MDName) { NamedMDNode *NamedMD = M->getNamedMetadata(MDName); std::set StrSet; if (!NamedMD) return StrSet; assert(NamedMD->getNumOperands() > 0 && "Invalid SPIR"); for (unsigned I = 0, E = NamedMD->getNumOperands(); I != E; ++I) { MDNode *MD = NamedMD->getOperand(I); if (!MD || MD->getNumOperands() == 0) continue; for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J) StrSet.insert(getMDOperandAsString(MD, J).str()); } return StrSet; } std::tuple getSPIRVSource(Module *M) { std::tuple Tup; if (auto N = SPIRVMDWalker(*M).getNamedMD(kSPIRVMD::Source).nextOp()) N.get(std::get<0>(Tup)) .get(std::get<1>(Tup)) .setQuiet(true) .get(std::get<2>(Tup)); return Tup; } ConstantInt *mapUInt(Module *M, ConstantInt *I, std::function F) { return ConstantInt::get(I->getType(), F(I->getZExtValue()), false); } ConstantInt *mapSInt(Module *M, ConstantInt *I, std::function F) { return ConstantInt::get(I->getType(), F(I->getSExtValue()), true); } bool isDecoratedSPIRVFunc(const Function *F, StringRef &UndecoratedName) { if (!F->hasName() || !F->getName().startswith(kSPIRVName::Prefix)) return false; UndecoratedName = F->getName(); return true; } /// Get TypePrimitiveEnum for special OpenCL type except opencl.block. SPIR::TypePrimitiveEnum getOCLTypePrimitiveEnum(StringRef TyName) { return StringSwitch(TyName) .Case("opencl.image1d_ro_t", SPIR::PRIMITIVE_IMAGE1D_RO_T) .Case("opencl.image1d_array_ro_t", SPIR::PRIMITIVE_IMAGE1D_ARRAY_RO_T) .Case("opencl.image1d_buffer_ro_t", SPIR::PRIMITIVE_IMAGE1D_BUFFER_RO_T) .Case("opencl.image2d_ro_t", SPIR::PRIMITIVE_IMAGE2D_RO_T) .Case("opencl.image2d_array_ro_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_RO_T) .Case("opencl.image2d_depth_ro_t", SPIR::PRIMITIVE_IMAGE2D_DEPTH_RO_T) .Case("opencl.image2d_array_depth_ro_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T) .Case("opencl.image2d_msaa_ro_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_RO_T) .Case("opencl.image2d_array_msaa_ro_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T) .Case("opencl.image2d_msaa_depth_ro_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T) .Case("opencl.image2d_array_msaa_depth_ro_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T) .Case("opencl.image3d_ro_t", SPIR::PRIMITIVE_IMAGE3D_RO_T) .Case("opencl.image1d_wo_t", SPIR::PRIMITIVE_IMAGE1D_WO_T) .Case("opencl.image1d_array_wo_t", SPIR::PRIMITIVE_IMAGE1D_ARRAY_WO_T) .Case("opencl.image1d_buffer_wo_t", SPIR::PRIMITIVE_IMAGE1D_BUFFER_WO_T) .Case("opencl.image2d_wo_t", SPIR::PRIMITIVE_IMAGE2D_WO_T) .Case("opencl.image2d_array_wo_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_WO_T) .Case("opencl.image2d_depth_wo_t", SPIR::PRIMITIVE_IMAGE2D_DEPTH_WO_T) .Case("opencl.image2d_array_depth_wo_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T) .Case("opencl.image2d_msaa_wo_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_WO_T) .Case("opencl.image2d_array_msaa_wo_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T) .Case("opencl.image2d_msaa_depth_wo_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T) .Case("opencl.image2d_array_msaa_depth_wo_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T) .Case("opencl.image3d_wo_t", SPIR::PRIMITIVE_IMAGE3D_WO_T) .Case("opencl.image1d_rw_t", SPIR::PRIMITIVE_IMAGE1D_RW_T) .Case("opencl.image1d_array_rw_t", SPIR::PRIMITIVE_IMAGE1D_ARRAY_RW_T) .Case("opencl.image1d_buffer_rw_t", SPIR::PRIMITIVE_IMAGE1D_BUFFER_RW_T) .Case("opencl.image2d_rw_t", SPIR::PRIMITIVE_IMAGE2D_RW_T) .Case("opencl.image2d_array_rw_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_RW_T) .Case("opencl.image2d_depth_rw_t", SPIR::PRIMITIVE_IMAGE2D_DEPTH_RW_T) .Case("opencl.image2d_array_depth_rw_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T) .Case("opencl.image2d_msaa_rw_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_RW_T) .Case("opencl.image2d_array_msaa_rw_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T) .Case("opencl.image2d_msaa_depth_rw_t", SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T) .Case("opencl.image2d_array_msaa_depth_rw_t", SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T) .Case("opencl.image3d_rw_t", SPIR::PRIMITIVE_IMAGE3D_RW_T) .Case("opencl.event_t", SPIR::PRIMITIVE_EVENT_T) .Case("opencl.pipe_ro_t", SPIR::PRIMITIVE_PIPE_RO_T) .Case("opencl.pipe_wo_t", SPIR::PRIMITIVE_PIPE_WO_T) .Case("opencl.reserve_id_t", SPIR::PRIMITIVE_RESERVE_ID_T) .Case("opencl.queue_t", SPIR::PRIMITIVE_QUEUE_T) .Case("opencl.clk_event_t", SPIR::PRIMITIVE_CLK_EVENT_T) .Case("opencl.sampler_t", SPIR::PRIMITIVE_SAMPLER_T) .Case("struct.ndrange_t", SPIR::PRIMITIVE_NDRANGE_T) .Case("opencl.intel_sub_group_avc_mce_payload_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_MCE_PAYLOAD_T) .Case("opencl.intel_sub_group_avc_ime_payload_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_PAYLOAD_T) .Case("opencl.intel_sub_group_avc_ref_payload_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_REF_PAYLOAD_T) .Case("opencl.intel_sub_group_avc_sic_payload_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_SIC_PAYLOAD_T) .Case("opencl.intel_sub_group_avc_mce_result_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_MCE_RESULT_T) .Case("opencl.intel_sub_group_avc_ime_result_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_RESULT_T) .Case("opencl.intel_sub_group_avc_ref_result_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_REF_RESULT_T) .Case("opencl.intel_sub_group_avc_sic_result_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_SIC_RESULT_T) .Case( "opencl.intel_sub_group_avc_ime_result_single_reference_streamout_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMOUT_T) .Case("opencl.intel_sub_group_avc_ime_result_dual_reference_streamout_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMOUT_T) .Case("opencl.intel_sub_group_avc_ime_single_reference_streamin_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_SINGLE_REF_STREAMIN_T) .Case("opencl.intel_sub_group_avc_ime_dual_reference_streamin_t", SPIR::PRIMITIVE_SUB_GROUP_AVC_IME_DUAL_REF_STREAMIN_T) .Default(SPIR::PRIMITIVE_NONE); } /// Translates LLVM type to descriptor for mangler. /// \param Signed indicates integer type should be translated as signed. /// \param VoidPtr indicates i8* should be translated as void*. static SPIR::RefParamType transTypeDesc(Type *Ty, const BuiltinArgTypeMangleInfo &Info) { bool Signed = Info.IsSigned; unsigned Attr = Info.Attr; bool VoidPtr = Info.IsVoidPtr; if (Info.IsEnum) return SPIR::RefParamType(new SPIR::PrimitiveType(Info.Enum)); if (Info.IsSampler) return SPIR::RefParamType( new SPIR::PrimitiveType(SPIR::PRIMITIVE_SAMPLER_T)); if (Info.IsAtomic && !Ty->isPointerTy()) { BuiltinArgTypeMangleInfo DTInfo = Info; DTInfo.IsAtomic = false; return SPIR::RefParamType(new SPIR::AtomicType(transTypeDesc(Ty, DTInfo))); } if (auto *IntTy = dyn_cast(Ty)) { switch (IntTy->getBitWidth()) { case 1: return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_BOOL)); case 8: return SPIR::RefParamType(new SPIR::PrimitiveType( Signed ? SPIR::PRIMITIVE_CHAR : SPIR::PRIMITIVE_UCHAR)); case 16: return SPIR::RefParamType(new SPIR::PrimitiveType( Signed ? SPIR::PRIMITIVE_SHORT : SPIR::PRIMITIVE_USHORT)); case 32: return SPIR::RefParamType(new SPIR::PrimitiveType( Signed ? SPIR::PRIMITIVE_INT : SPIR::PRIMITIVE_UINT)); case 64: return SPIR::RefParamType(new SPIR::PrimitiveType( Signed ? SPIR::PRIMITIVE_LONG : SPIR::PRIMITIVE_ULONG)); default: return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_INT)); } } if (Ty->isVoidTy()) return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_VOID)); if (Ty->isHalfTy()) return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_HALF)); if (Ty->isFloatTy()) return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_FLOAT)); if (Ty->isDoubleTy()) return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_DOUBLE)); if (Ty->isBFloatTy()) return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_BFLOAT)); if (auto *VecTy = dyn_cast(Ty)) { return SPIR::RefParamType(new SPIR::VectorType( transTypeDesc(VecTy->getElementType(), Info), VecTy->getNumElements())); } if (Ty->isArrayTy()) { BuiltinArgTypeMangleInfo DTInfo = Info; DTInfo.PointerElementType.setPointer(Ty->getArrayElementType()); return transTypeDesc(Ty->getArrayElementType()->getPointerTo(0), DTInfo); } if (Ty->isStructTy()) { auto Name = Ty->getStructName(); std::string Tmp; if (Name.startswith(kLLVMTypeName::StructPrefix)) Name = Name.drop_front(strlen(kLLVMTypeName::StructPrefix)); if (Name.startswith(kSPIRVTypeName::PrefixAndDelim)) { Name = Name.substr(sizeof(kSPIRVTypeName::PrefixAndDelim) - 1); Tmp = Name.str(); auto Pos = Tmp.find(kSPIRVTypeName::Delimiter); // first dot while (Pos != std::string::npos) { Tmp[Pos] = '_'; Pos = Tmp.find(kSPIRVTypeName::Delimiter, Pos); } Name = Tmp = kSPIRVName::Prefix + Tmp; } // ToDo: Create a better unique Name for struct without Name if (Name.empty()) { std::ostringstream OS; OS << reinterpret_cast(Ty); Name = Tmp = std::string("struct_") + OS.str(); } return SPIR::RefParamType(new SPIR::UserDefinedType(Name.str())); } if (Ty->isPointerTy()) { auto *ET = Info.PointerElementType.getPointer(); if (!ET) ET = Type::getInt8Ty(Ty->getContext()); BuiltinArgTypeMangleInfo DTInfo = Info; if (Info.PointerElementType.getInt()) { ET = DTInfo.PointerElementType.getPointer()->getPointerTo(0); DTInfo.PointerElementType.setInt(false); } else { DTInfo.PointerElementType.setPointer(Type::getInt8Ty(Ty->getContext())); } SPIR::ParamType *EPT = nullptr; if (isa(ET)) { assert(isVoidFuncTy(cast(ET)) && "Not supported"); EPT = new SPIR::BlockType; } else if (auto StructTy = dyn_cast(ET)) { LLVM_DEBUG(dbgs() << "ptr to struct: " << *Ty << '\n'); auto TyName = StructTy->getStructName(); if (TyName.startswith(kSPR2TypeName::OCLPrefix)) { auto DelimPos = TyName.find_first_of(kSPR2TypeName::Delimiter, strlen(kSPR2TypeName::OCLPrefix)); if (DelimPos != StringRef::npos) TyName = TyName.substr(0, DelimPos); } LLVM_DEBUG(dbgs() << " type Name: " << TyName << '\n'); auto Prim = getOCLTypePrimitiveEnum(TyName); if (StructTy->isOpaque()) { if (TyName == "opencl.block") { auto BlockTy = new SPIR::BlockType; // Handle block with local memory arguments according to OpenCL 2.0 // spec. if (Info.IsLocalArgBlock) { SPIR::RefParamType VoidTyRef( new SPIR::PrimitiveType(SPIR::PRIMITIVE_VOID)); auto VoidPtrTy = new SPIR::PointerType(VoidTyRef); VoidPtrTy->setAddressSpace(SPIR::ATTR_LOCAL); // "__local void *" BlockTy->setParam(0, SPIR::RefParamType(VoidPtrTy)); // "..." BlockTy->setParam(1, SPIR::RefParamType(new SPIR::PrimitiveType( SPIR::PRIMITIVE_VAR_ARG))); } EPT = BlockTy; } else if (Prim != SPIR::PRIMITIVE_NONE) { if (Prim == SPIR::PRIMITIVE_PIPE_RO_T || Prim == SPIR::PRIMITIVE_PIPE_WO_T) { SPIR::RefParamType OpaqueTyRef(new SPIR::PrimitiveType(Prim)); auto OpaquePtrTy = new SPIR::PointerType(OpaqueTyRef); OpaquePtrTy->setAddressSpace(getOCLOpaqueTypeAddrSpace(Prim)); EPT = OpaquePtrTy; } else { EPT = new SPIR::PrimitiveType(Prim); } } } else if (Prim == SPIR::PRIMITIVE_NDRANGE_T) // ndrange_t is not opaque type EPT = new SPIR::PrimitiveType(SPIR::PRIMITIVE_NDRANGE_T); } if (EPT) return SPIR::RefParamType(EPT); if (VoidPtr && ET->isIntegerTy(8)) ET = Type::getVoidTy(ET->getContext()); auto *PT = new SPIR::PointerType(transTypeDesc(ET, DTInfo)); PT->setAddressSpace(static_cast( Ty->getPointerAddressSpace() + (unsigned)SPIR::ATTR_ADDR_SPACE_FIRST)); for (unsigned I = SPIR::ATTR_QUALIFIER_FIRST, E = SPIR::ATTR_QUALIFIER_LAST; I <= E; ++I) PT->setQualifier(static_cast(I), I & Attr); return SPIR::RefParamType(PT); } LLVM_DEBUG(dbgs() << "[transTypeDesc] " << *Ty << '\n'); assert(0 && "not implemented"); return SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_INT)); } Value *getScalarOrArray(Value *V, unsigned Size, Instruction *Pos) { if (!V->getType()->isPointerTy()) return V; auto GEP = cast(V); assert(GEP->getNumOperands() == 3 && "must be a GEP from an array"); assert(GEP->getSourceElementType()->getArrayNumElements() == Size); assert(dyn_cast(GEP->getOperand(1))->getZExtValue() == 0); assert(dyn_cast(GEP->getOperand(2))->getZExtValue() == 0); return new LoadInst(GEP->getSourceElementType(), GEP->getOperand(0), "", Pos); } Constant *getScalarOrVectorConstantInt(Type *T, uint64_t V, bool IsSigned) { if (auto IT = dyn_cast(T)) return ConstantInt::get(IT, V); if (auto VT = dyn_cast(T)) { std::vector EV( VT->getNumElements(), getScalarOrVectorConstantInt(VT->getElementType(), V, IsSigned)); return ConstantVector::get(EV); } llvm_unreachable("Invalid type"); return nullptr; } Value *getScalarOrArrayConstantInt(Instruction *Pos, Type *T, unsigned Len, uint64_t V, bool IsSigned) { if (auto IT = dyn_cast(T)) { assert(Len == 1 && "Invalid length"); return ConstantInt::get(IT, V, IsSigned); } if (isa(T)) { unsigned PointerSize = Pos->getModule()->getDataLayout().getPointerTypeSizeInBits(T); auto *ET = Type::getIntNTy(T->getContext(), PointerSize); assert(cast(T)->isOpaqueOrPointeeTypeMatches(ET) && "Pointer-to-non-size_t arguments are not valid for this call"); auto AT = ArrayType::get(ET, Len); std::vector EV(Len, ConstantInt::get(ET, V, IsSigned)); auto CA = ConstantArray::get(AT, EV); auto Alloca = new AllocaInst(AT, 0, "", Pos); new StoreInst(CA, Alloca, Pos); auto Zero = ConstantInt::getNullValue(Type::getInt32Ty(T->getContext())); Value *Index[] = {Zero, Zero}; auto *Ret = GetElementPtrInst::CreateInBounds(AT, Alloca, Index, "", Pos); LLVM_DEBUG(dbgs() << "[getScalarOrArrayConstantInt] Alloca: " << *Alloca << ", Return: " << *Ret << '\n'); return Ret; } if (auto AT = dyn_cast(T)) { auto ET = AT->getArrayElementType(); assert(AT->getArrayNumElements() == Len); std::vector EV(Len, ConstantInt::get(ET, V, IsSigned)); auto Ret = ConstantArray::get(AT, EV); LLVM_DEBUG(dbgs() << "[getScalarOrArrayConstantInt] Array type: " << *AT << ", Return: " << *Ret << '\n'); return Ret; } llvm_unreachable("Invalid type"); return nullptr; } void dumpUsers(Value *V, StringRef Prompt) { if (!V) return; LLVM_DEBUG(dbgs() << Prompt << " Users of " << *V << " :\n"); for (auto UI = V->user_begin(), UE = V->user_end(); UI != UE; ++UI) LLVM_DEBUG(dbgs() << " " << **UI << '\n'); } std::string getSPIRVTypeName(StringRef BaseName, StringRef Postfixes) { assert(!BaseName.empty() && "Invalid SPIR-V type Name"); auto TN = std::string(kSPIRVTypeName::PrefixAndDelim) + BaseName.str(); if (Postfixes.empty()) return TN; return TN + kSPIRVTypeName::Delimiter + Postfixes.str(); } bool isSPIRVConstantName(StringRef TyName) { if (TyName == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler) || TyName == getSPIRVTypeName(kSPIRVTypeName::ConstantPipeStorage)) return true; return false; } Type *getSPIRVTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName) { return PointerType::get( getSPIRVStructTypeByChangeBaseTypeName(M, T, OldName, NewName), SPIRAS_Global); } Type *getSPIRVStructTypeByChangeBaseTypeName(Module *M, Type *T, StringRef OldName, StringRef NewName) { StringRef Postfixes; if (isSPIRVStructType(T, OldName, &Postfixes)) return getOrCreateOpaqueStructType(M, getSPIRVTypeName(NewName, Postfixes)); LLVM_DEBUG(dbgs() << " Invalid SPIR-V type " << *T << '\n'); llvm_unreachable("Invalid SPIR-V type"); return nullptr; } std::string getSPIRVImageTypePostfixes(StringRef SampledType, SPIRVTypeImageDescriptor Desc, SPIRVAccessQualifierKind Acc) { std::string S; raw_string_ostream OS(S); OS << kSPIRVTypeName::PostfixDelim << SampledType << kSPIRVTypeName::PostfixDelim << Desc.Dim << kSPIRVTypeName::PostfixDelim << Desc.Depth << kSPIRVTypeName::PostfixDelim << Desc.Arrayed << kSPIRVTypeName::PostfixDelim << Desc.MS << kSPIRVTypeName::PostfixDelim << Desc.Sampled << kSPIRVTypeName::PostfixDelim << Desc.Format << kSPIRVTypeName::PostfixDelim << Acc; return OS.str(); } std::string getSPIRVImageSampledTypeName(SPIRVType *Ty) { switch (Ty->getOpCode()) { case OpTypeVoid: return kSPIRVImageSampledTypeName::Void; case OpTypeInt: if (Ty->getIntegerBitWidth() == 32) { if (static_cast(Ty)->isSigned()) return kSPIRVImageSampledTypeName::Int; else return kSPIRVImageSampledTypeName::UInt; } if (Ty->getIntegerBitWidth() == 64) { if (static_cast(Ty)->isSigned()) return kSPIRVImageSampledTypeName::Long; return kSPIRVImageSampledTypeName::ULong; } break; case OpTypeFloat: switch (Ty->getFloatBitWidth()) { case 16: return kSPIRVImageSampledTypeName::Half; case 32: return kSPIRVImageSampledTypeName::Float; default: break; } break; default: break; } llvm_unreachable("Invalid sampled type for image"); return std::string(); } // ToDo: Find a way to represent uint sampled type in LLVM, maybe an // opaque type. Type *getLLVMTypeForSPIRVImageSampledTypePostfix(StringRef Postfix, LLVMContext &Ctx) { if (Postfix == kSPIRVImageSampledTypeName::Void) return Type::getVoidTy(Ctx); if (Postfix == kSPIRVImageSampledTypeName::Float) return Type::getFloatTy(Ctx); if (Postfix == kSPIRVImageSampledTypeName::Half) return Type::getHalfTy(Ctx); if (Postfix == kSPIRVImageSampledTypeName::Int || Postfix == kSPIRVImageSampledTypeName::UInt) return Type::getInt32Ty(Ctx); if (Postfix == kSPIRVImageSampledTypeName::Long || Postfix == kSPIRVImageSampledTypeName::ULong) return Type::getInt64Ty(Ctx); llvm_unreachable("Invalid sampled type postfix"); return nullptr; } std::string getImageBaseTypeName(StringRef Name) { SmallVector SubStrs; const char Delims[] = {kSPR2TypeName::Delimiter, 0}; Name.split(SubStrs, Delims); if (Name.startswith(kSPR2TypeName::OCLPrefix)) { Name = SubStrs[1]; } else { Name = SubStrs[0]; } std::string ImageTyName{Name}; if (hasAccessQualifiedName(Name)) ImageTyName.erase(ImageTyName.size() - 5, 3); return ImageTyName; } size_t getImageOperandsIndex(Op OpCode) { switch (OpCode) { case OpImageRead: case OpImageSampleExplicitLod: return 2; case OpImageWrite: return 3; default: return ~0U; } } std::string mapOCLTypeNameToSPIRV(StringRef Name, StringRef Acc) { std::string BaseTy; std::string Postfixes; raw_string_ostream OS(Postfixes); if (Name.startswith(kSPR2TypeName::ImagePrefix)) { std::string ImageTyName = getImageBaseTypeName(Name); auto Desc = map(ImageTyName); LLVM_DEBUG(dbgs() << "[trans image type] " << Name << " => " << "(" << (unsigned)Desc.Dim << ", " << Desc.Depth << ", " << Desc.Arrayed << ", " << Desc.MS << ", " << Desc.Sampled << ", " << Desc.Format << ")\n"); BaseTy = kSPIRVTypeName::Image; OS << getSPIRVImageTypePostfixes( kSPIRVImageSampledTypeName::Void, Desc, SPIRSPIRVAccessQualifierMap::map(Acc.str())); } else { LLVM_DEBUG(dbgs() << "Mapping of " << Name << " is not implemented\n"); llvm_unreachable("Not implemented"); } return getSPIRVTypeName(BaseTy, OS.str()); } bool eraseIfNoUse(Function *F) { bool Changed = false; if (!F) return Changed; if (!GlobalValue::isInternalLinkage(F->getLinkage()) && !F->isDeclaration()) return Changed; dumpUsers(F, "[eraseIfNoUse] "); for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE;) { auto U = *UI++; if (auto CE = dyn_cast(U)) { if (CE->use_empty()) { CE->dropAllReferences(); Changed = true; } } } if (F->use_empty()) { LLVM_DEBUG(dbgs() << "Erase "; F->printAsOperand(dbgs()); dbgs() << '\n'); F->eraseFromParent(); Changed = true; } return Changed; } void eraseIfNoUse(Value *V) { if (!V->use_empty()) return; if (Constant *C = dyn_cast(V)) { C->destroyConstant(); return; } if (Instruction *I = dyn_cast(V)) { if (!I->mayHaveSideEffects()) I->eraseFromParent(); } eraseIfNoUse(dyn_cast(V)); } bool eraseUselessFunctions(Module *M) { bool Changed = false; for (auto I = M->begin(), E = M->end(); I != E;) Changed |= eraseIfNoUse(&(*I++)); return Changed; } // The mangling algorithm follows OpenCL pipe built-ins clang 3.8 CodeGen rules. static SPIR::MangleError manglePipeOrAddressSpaceCastBuiltin(const SPIR::FunctionDescriptor &Fd, std::string &MangledName) { assert(OCLUtil::isPipeOrAddressSpaceCastBI(Fd.Name) && "Method is expected to be called only for pipe and address space cast " "builtins!"); if (Fd.isNull()) { MangledName.assign(SPIR::FunctionDescriptor::nullString()); return SPIR::MANGLE_NULL_FUNC_DESCRIPTOR; } MangledName.assign("__" + Fd.Name); return SPIR::MANGLE_SUCCESS; } std::string mangleBuiltin(StringRef UniqName, ArrayRef ArgTypes, BuiltinFuncMangleInfo *BtnInfo) { if (!BtnInfo) return std::string(UniqName); BtnInfo->init(UniqName); if (BtnInfo->avoidMangling()) return std::string(UniqName); std::string MangledName; LLVM_DEBUG(dbgs() << "[mangle] " << UniqName << " => "); SPIR::FunctionDescriptor FD; FD.Name = BtnInfo->getUnmangledName(); bool BIVarArgNegative = BtnInfo->getVarArg() < 0; if (ArgTypes.empty()) { // Function signature cannot be ()(void, ...) so if there is an ellipsis // it must be ()(...) if (BIVarArgNegative) { FD.Parameters.emplace_back( SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_VOID))); } } else { for (unsigned I = 0, E = BIVarArgNegative ? ArgTypes.size() : (unsigned)BtnInfo->getVarArg(); I != E; ++I) { auto T = ArgTypes[I]; FD.Parameters.emplace_back( transTypeDesc(T, BtnInfo->getTypeMangleInfo(I))); } } // Ellipsis must be the last argument of any function if (!BIVarArgNegative) { assert((unsigned)BtnInfo->getVarArg() <= ArgTypes.size() && "invalid index of an ellipsis"); FD.Parameters.emplace_back( SPIR::RefParamType(new SPIR::PrimitiveType(SPIR::PRIMITIVE_VAR_ARG))); } #if defined(SPIRV_SPIR20_MANGLING_REQUIREMENTS) SPIR::NameMangler Mangler(SPIR::SPIR20); Mangler.mangle(FD, MangledName); #else if (OCLUtil::isPipeOrAddressSpaceCastBI(BtnInfo->getUnmangledName())) { manglePipeOrAddressSpaceCastBuiltin(FD, MangledName); } else { SPIR::NameMangler Mangler(SPIR::SPIR20); Mangler.mangle(FD, MangledName); } #endif LLVM_DEBUG(dbgs() << MangledName << '\n'); return MangledName; } /// Check if access qualifier is encoded in the type Name. bool hasAccessQualifiedName(StringRef TyName) { if (TyName.size() < 5) return false; auto Acc = TyName.substr(TyName.size() - 5, 3); return llvm::StringSwitch(Acc) .Case(kAccessQualPostfix::ReadOnly, true) .Case(kAccessQualPostfix::WriteOnly, true) .Case(kAccessQualPostfix::ReadWrite, true) .Default(false); } SPIRVAccessQualifierKind getAccessQualifier(StringRef TyName) { return SPIRSPIRVAccessQualifierMap::map( getAccessQualifierFullName(TyName).str()); } StringRef getAccessQualifierPostfix(SPIRVAccessQualifierKind Access) { switch (Access) { case AccessQualifierReadOnly: return kAccessQualPostfix::ReadOnly; case AccessQualifierWriteOnly: return kAccessQualPostfix::WriteOnly; case AccessQualifierReadWrite: return kAccessQualPostfix::ReadWrite; default: assert(false && "Unrecognized access qualifier!"); return kAccessQualPostfix::ReadWrite; } } /// Get access qualifier from the type Name. StringRef getAccessQualifierFullName(StringRef TyName) { assert(hasAccessQualifiedName(TyName) && "Type is not qualified with access."); auto Acc = TyName.substr(TyName.size() - 5, 3); return llvm::StringSwitch(Acc) .Case(kAccessQualPostfix::ReadOnly, kAccessQualName::ReadOnly) .Case(kAccessQualPostfix::WriteOnly, kAccessQualName::WriteOnly) .Case(kAccessQualPostfix::ReadWrite, kAccessQualName::ReadWrite); } /// Translates OpenCL image type names to SPIR-V. Type *adaptSPIRVImageType(Module *M, Type *PointeeType) { if (isOCLImageStructType(PointeeType)) { auto ImageTypeName = PointeeType->getStructName(); StringRef Acc = kAccessQualName::ReadOnly; if (hasAccessQualifiedName(ImageTypeName)) Acc = getAccessQualifierFullName(ImageTypeName); return getOrCreateOpaqueStructType( M, mapOCLTypeNameToSPIRV(ImageTypeName, Acc)); } return PointeeType; } llvm::PointerType *getOCLClkEventType(Module *M) { return getOrCreateOpaquePtrType(M, SPIR_TYPE_NAME_CLK_EVENT_T, SPIRAS_Private); } llvm::PointerType *getOCLClkEventPtrType(Module *M) { return PointerType::get(getOCLClkEventType(M), SPIRAS_Generic); } llvm::Constant *getOCLNullClkEventPtr(Module *M) { return Constant::getNullValue(getOCLClkEventPtrType(M)); } bool hasLoopMetadata(const Module *M) { for (const Function &F : *M) for (const BasicBlock &BB : F) { const Instruction *Term = BB.getTerminator(); if (Term && Term->getMetadata("llvm.loop")) return true; } return false; } bool isSPIRVOCLExtInst(const CallInst *CI, OCLExtOpKind *ExtOp) { StringRef DemangledName; if (!oclIsBuiltin(CI->getCalledFunction()->getName(), DemangledName)) return false; StringRef S = DemangledName; if (!S.startswith(kSPIRVName::Prefix)) return false; S = S.drop_front(strlen(kSPIRVName::Prefix)); auto Loc = S.find(kSPIRVPostfix::Divider); auto ExtSetName = S.substr(0, Loc); SPIRVExtInstSetKind Set = SPIRVEIS_Count; if (!SPIRVExtSetShortNameMap::rfind(ExtSetName.str(), &Set)) return false; if (Set != SPIRVEIS_OpenCL) return false; auto ExtOpName = S.substr(Loc + 1); auto PostFixPos = ExtOpName.find("_R"); ExtOpName = ExtOpName.substr(0, PostFixPos); OCLExtOpKind EOC; if (!OCLExtOpMap::rfind(ExtOpName.str(), &EOC)) return false; *ExtOp = EOC; return true; } std::string decodeSPIRVTypeName(StringRef Name, SmallVectorImpl &Strs) { SmallVector SubStrs; const char Delim[] = {kSPIRVTypeName::Delimiter, 0}; Name.split(SubStrs, Delim, -1, true); assert(SubStrs.size() >= 2 && "Invalid SPIRV type name"); assert(SubStrs[0] == kSPIRVTypeName::Prefix && "Invalid prefix"); assert((SubStrs.size() == 2 || !SubStrs[2].empty()) && "Invalid postfix"); if (SubStrs.size() > 2) { const char PostDelim[] = {kSPIRVTypeName::PostfixDelim, 0}; SmallVector Postfixes; SubStrs[2].split(Postfixes, PostDelim, -1, true); assert(Postfixes.size() > 1 && Postfixes[0].empty() && "Invalid postfix"); for (unsigned I = 1, E = Postfixes.size(); I != E; ++I) Strs.push_back(std::string(Postfixes[I]).c_str()); } return SubStrs[1].str(); } // Returns true if type(s) and number of elements (if vector) is valid bool checkTypeForSPIRVExtendedInstLowering(IntrinsicInst *II, SPIRVModule *BM) { switch (II->getIntrinsicID()) { case Intrinsic::ceil: case Intrinsic::copysign: case Intrinsic::cos: case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::fabs: case Intrinsic::floor: case Intrinsic::fma: case Intrinsic::log: case Intrinsic::log10: case Intrinsic::log2: case Intrinsic::maximum: case Intrinsic::maxnum: case Intrinsic::minimum: case Intrinsic::minnum: case Intrinsic::nearbyint: case Intrinsic::pow: case Intrinsic::powi: case Intrinsic::rint: case Intrinsic::round: case Intrinsic::roundeven: case Intrinsic::sin: case Intrinsic::sqrt: case Intrinsic::trunc: { // Although some of the intrinsics above take multiple arguments, it is // sufficient to check arg 0 because the LLVM Verifier will have checked // that all floating point operands have the same type and the second // argument of powi is i32. Type *Ty = II->getType(); if (II->getArgOperand(0)->getType() != Ty) return false; int NumElems = 1; if (auto *VecTy = dyn_cast(Ty)) { NumElems = VecTy->getNumElements(); Ty = VecTy->getElementType(); } if (Ty->isBFloatTy() && BM->hasCapability(internal::CapabilityBFloat16ArithmeticINTEL)) return true; if ((!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy()) || (!BM->hasCapability(CapabilityVectorAnyINTEL) && ((NumElems > 4) && (NumElems != 8) && (NumElems != 16)))) { BM->SPIRVCK( false, InvalidFunctionCall, II->getCalledOperand()->getName().str()); return false; } break; } case Intrinsic::abs: { Type *Ty = II->getType(); int NumElems = 1; if (auto *VecTy = dyn_cast(Ty)) { NumElems = VecTy->getNumElements(); Ty = VecTy->getElementType(); } if (Ty->isBFloatTy() && BM->hasCapability(internal::CapabilityBFloat16ArithmeticINTEL)) return true; if ((!Ty->isIntegerTy()) || (!BM->hasCapability(CapabilityVectorAnyINTEL) && ((NumElems > 4) && (NumElems != 8) && (NumElems != 16)))) { BM->SPIRVCK( false, InvalidFunctionCall, II->getCalledOperand()->getName().str()); } break; } default: break; } return true; } CallInst *setAttrByCalledFunc(CallInst *Call) { Function *F = Call->getCalledFunction(); assert(F); if (F->isIntrinsic()) { return Call; } Call->setCallingConv(F->getCallingConv()); Call->setAttributes(F->getAttributes()); return Call; } bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind) { if (!GV->hasName() || !getSPIRVBuiltin(GV->getName().str(), *Kind)) return false; return true; } // Variable like GlobalInvolcationId[x] -> get_global_id(x). // Variable like WorkDim -> get_work_dim(). // Replace the following pattern: // %a = addrspacecast i32 addrspace(1)* @__spirv_BuiltInSubgroupMaxSize to // i32 addrspace(4)* // %b = load i32, i32 addrspace(4)* %a, align 4 // %c = load i32, i32 addrspace(4)* %a, align 4 // With: // %b = call spir_func i32 @_Z22get_max_sub_group_sizev() // %c = call spir_func i32 @_Z22get_max_sub_group_sizev() // And replace the following pattern: // %a = addrspacecast <3 x i64> addrspace(1)* @__spirv_BuiltInWorkgroupId to // <3 x i64> addrspace(4)* // %b = load <3 x i64>, <3 x i64> addrspace(4)* %a, align 32 // %c = extractelement <3 x i64> %b, i32 idx // %d = extractelement <3 x i64> %b, i32 idx // With: // %0 = call spir_func i64 @_Z13get_global_idj(i32 0) #1 // %1 = insertelement <3 x i64> undef, i64 %0, i32 0 // %2 = call spir_func i64 @_Z13get_global_idj(i32 1) #1 // %3 = insertelement <3 x i64> %1, i64 %2, i32 1 // %4 = call spir_func i64 @_Z13get_global_idj(i32 2) #1 // %5 = insertelement <3 x i64> %3, i64 %4, i32 2 // %c = extractelement <3 x i64> %5, i32 idx // %d = extractelement <3 x i64> %5, i32 idx // // Replace the following pattern: // %0 = addrspacecast <3 x i64> addrspace(1)* @__spirv_BuiltInWorkgroupSize to // <3 x i64> addrspace(4)* // %1 = getelementptr <3 x i64>, <3 x i64> addrspace(4)* %0, i64 0, i64 0 // %2 = load i64, i64 addrspace(4)* %1, align 32 // With: // %0 = call spir_func i64 @_Z13get_global_idj(i32 0) #1 // %1 = insertelement <3 x i64> undef, i64 %0, i32 0 // %2 = call spir_func i64 @_Z13get_global_idj(i32 1) #1 // %3 = insertelement <3 x i64> %1, i64 %2, i32 1 // %4 = call spir_func i64 @_Z13get_global_idj(i32 2) #1 // %5 = insertelement <3 x i64> %3, i64 %4, i32 2 // %6 = extractelement <3 x i64> %5, i32 0 /// Recursively look through the uses of a global variable, including casts or /// gep offsets, to find all loads of the variable. Gep offsets that are non-0 /// are accumulated in the AccumulatedOffset parameter, which will eventually be /// used to figure out which index of a variable is being used. static void replaceUsesOfBuiltinVar(Value *V, const APInt &AccumulatedOffset, Function *ReplacementFunc, GlobalVariable *GV) { const DataLayout &DL = ReplacementFunc->getParent()->getDataLayout(); SmallVector InstsToRemove; for (User *U : V->users()) { if (auto *Cast = dyn_cast(U)) { replaceUsesOfBuiltinVar(Cast, AccumulatedOffset, ReplacementFunc, GV); InstsToRemove.push_back(Cast); } else if (auto *GEP = dyn_cast(U)) { APInt NewOffset = AccumulatedOffset.sextOrTrunc( DL.getIndexSizeInBits(GEP->getPointerAddressSpace())); if (!GEP->accumulateConstantOffset(DL, NewOffset)) llvm_unreachable("Illegal GEP of a SPIR-V builtin variable"); replaceUsesOfBuiltinVar(GEP, NewOffset, ReplacementFunc, GV); InstsToRemove.push_back(GEP); } else if (auto *Load = dyn_cast(U)) { // Figure out which index the accumulated offset corresponds to. If we // have a weird offset (e.g., trying to load byte 7), bail out. Type *ScalarTy = ReplacementFunc->getReturnType(); APInt Index; uint64_t Remainder; APInt::udivrem(AccumulatedOffset, ScalarTy->getScalarSizeInBits() / 8, Index, Remainder); if (Remainder != 0) llvm_unreachable("Illegal GEP of a SPIR-V builtin variable"); IRBuilder<> Builder(Load); Value *Replacement; if (ReplacementFunc->getFunctionType()->getNumParams() == 0) { if (Load->getType() != ScalarTy) llvm_unreachable("Illegal use of a SPIR-V builtin variable"); Replacement = setAttrByCalledFunc(Builder.CreateCall(ReplacementFunc, {})); } else { // The function has an index parameter. if (auto *VecTy = dyn_cast(Load->getType())) { // Reconstruct the original global variable vector because // the load type may not match. // global <3 x i64>, load <6 x i32> VecTy = cast(GV->getValueType()); if (!Index.isZero() || DL.getTypeSizeInBits(VecTy) != DL.getTypeSizeInBits(Load->getType())) llvm_unreachable("Illegal use of a SPIR-V builtin variable"); Replacement = UndefValue::get(VecTy); for (unsigned I = 0; I < VecTy->getNumElements(); I++) { Replacement = Builder.CreateInsertElement( Replacement, setAttrByCalledFunc( Builder.CreateCall(ReplacementFunc, {Builder.getInt32(I)})), Builder.getInt32(I)); } // Insert a bitcast from the reconstructed vector to the load vector // type in case they are different. // Input: // %1 = load <6 x i32>, ptr addrspace(1) %0, align 32 // %2 = extractelement <6 x i32> %1, i32 0 // %3 = add i32 5, %2 // Modified: // < reconstruct global vector elements 0 and 1 > // %2 = insertelement <3 x i64> %0, i64 %1, i32 2 // %3 = bitcast <3 x i64> %2 to <6 x i32> // %4 = extractelement <6 x i32> %3, i32 0 // %5 = add i32 5, %4 Replacement = Builder.CreateBitCast(Replacement, Load->getType()); } else if (Load->getType() == ScalarTy) { Replacement = setAttrByCalledFunc(Builder.CreateCall( ReplacementFunc, {Builder.getInt32(Index.getZExtValue())})); } else { llvm_unreachable("Illegal load type of a SPIR-V builtin variable"); } } Load->replaceAllUsesWith(Replacement); InstsToRemove.push_back(Load); } else { llvm_unreachable("Illegal use of a SPIR-V builtin variable"); } } for (Instruction *I : InstsToRemove) I->eraseFromParent(); } bool lowerBuiltinVariableToCall(GlobalVariable *GV, SPIRVBuiltinVariableKind Kind) { // There might be dead constant users of GV (for example, SPIRVLowerConstExpr // replaces ConstExpr uses but those ConstExprs are not deleted, since LLVM // constants are created on demand as needed and never deleted). // Remove them first! GV->removeDeadConstantUsers(); Module *M = GV->getParent(); LLVMContext &C = M->getContext(); std::string FuncName = GV->getName().str(); Type *GVTy = GV->getValueType(); Type *ReturnTy = GVTy; // Some SPIR-V builtin variables are translated to a function with an index // argument. bool HasIndexArg = ReturnTy->isVectorTy() && !(BuiltInSubgroupEqMask <= Kind && Kind <= BuiltInSubgroupLtMask); if (HasIndexArg) ReturnTy = cast(ReturnTy)->getElementType(); std::vector ArgTy; if (HasIndexArg) ArgTy.push_back(Type::getInt32Ty(C)); std::string MangledName; mangleOpenClBuiltin(FuncName, ArgTy, {}, MangledName); Function *Func = M->getFunction(MangledName); if (!Func) { FunctionType *FT = FunctionType::get(ReturnTy, ArgTy, false); Func = Function::Create(FT, GlobalValue::ExternalLinkage, MangledName, M); Func->setCallingConv(CallingConv::SPIR_FUNC); Func->addFnAttr(Attribute::NoUnwind); Func->addFnAttr(Attribute::WillReturn); Func->setDoesNotAccessMemory(); } replaceUsesOfBuiltinVar(GV, APInt(64, 0), Func, GV); return true; } bool lowerBuiltinVariablesToCalls(Module *M) { std::vector WorkList; for (auto I = M->global_begin(), E = M->global_end(); I != E; ++I) { SPIRVBuiltinVariableKind Kind; if (!isSPIRVBuiltinVariable(&(*I), &Kind)) continue; if (!lowerBuiltinVariableToCall(&(*I), Kind)) return false; WorkList.push_back(&(*I)); } for (auto &I : WorkList) { I->eraseFromParent(); } return true; } /// Transforms SPV-IR work-item builtin calls to SPIRV builtin variables. /// e.g. /// SPV-IR: @_Z33__spirv_BuiltInGlobalInvocationIdi(i) /// is transformed as: /// x = load GlobalInvocationId; extract x, i /// e.g. /// SPV-IR: @_Z22__spirv_BuiltInWorkDim() /// is transformed as: /// load WorkDim bool lowerBuiltinCallsToVariables(Module *M) { LLVM_DEBUG(dbgs() << "Enter lowerBuiltinCallsToVariables\n"); // Store instructions and functions that need to be removed. SmallVector ToRemove; for (auto &F : *M) { // Builtins should be declaration only. if (!F.isDeclaration()) continue; StringRef DemangledName; if (!oclIsBuiltin(F.getName(), DemangledName)) continue; LLVM_DEBUG(dbgs() << "Function demangled name: " << DemangledName << '\n'); SmallVector Postfix; // Deprefix "__spirv_" StringRef Name = dePrefixSPIRVName(DemangledName, Postfix); // Lookup SPIRV Builtin map. if (!SPIRVBuiltInNameMap::rfind(Name.str(), nullptr)) continue; std::string BuiltinVarName = DemangledName.str(); LLVM_DEBUG(dbgs() << "builtin variable name: " << BuiltinVarName << '\n'); bool IsVec = F.getFunctionType()->getNumParams() > 0; Type *GVType = IsVec ? FixedVectorType::get(F.getReturnType(), 3) : F.getReturnType(); GlobalVariable *BV = nullptr; // Consider the following LLVM IR: // @__spirv_BuiltInLocalInvocationId = // ..... // define spir_kernel void @kernel1(....) { // %3 = tail call i64 @_Z12get_local_idj(i32 0) // ..... // return void // } // During the OCLToSPIRV pass, the opencl call will get lowered to // yet another global variable with the name // '@__spirv_BuiltInLocalInvocationId'. In such a case, we would want to // create only a single global variable with this name. if (GlobalVariable *GV = M->getGlobalVariable(BuiltinVarName)) BV = GV; else BV = new GlobalVariable(*M, GVType, /*isConstant=*/true, GlobalValue::ExternalLinkage, nullptr, BuiltinVarName, 0, GlobalVariable::NotThreadLocal, SPIRAS_Input); for (auto *U : F.users()) { auto *CI = dyn_cast(U); assert(CI && "invalid instruction"); IRBuilder<> Builder(CI); Value *NewValue = Builder.CreateLoad(GVType, BV); LLVM_DEBUG(dbgs() << "Transform: " << *CI << " => " << *NewValue << '\n'); if (IsVec) { auto *GVVecTy = cast(GVType); ConstantInt *Bound = Builder.getInt32(GVVecTy->getNumElements()); // Create a select on the index first, to avoid undefined behaviour // due to exceeding the vector size by the extractelement. Value *IndexCmp = Builder.CreateICmpULT(CI->getArgOperand(0), Bound); Constant *ZeroIndex = ConstantInt::get(CI->getArgOperand(0)->getType(), 0); Value *ExtractIndex = Builder.CreateSelect(IndexCmp, CI->getArgOperand(0), ZeroIndex); // Extract from builtin variable. NewValue = Builder.CreateExtractElement(NewValue, ExtractIndex); // Clamp to out-of-range value. Constant *OutOfRangeVal = ConstantInt::get( F.getReturnType(), getBuiltinOutOfRangeValue(BuiltinVarName)); NewValue = Builder.CreateSelect(IndexCmp, NewValue, OutOfRangeVal); LLVM_DEBUG(dbgs() << *NewValue << '\n'); } NewValue->takeName(CI); CI->replaceAllUsesWith(NewValue); ToRemove.push_back(CI); } ToRemove.push_back(&F); } for (auto *V : ToRemove) { if (auto *I = dyn_cast(V)) I->eraseFromParent(); else if (auto *F = dyn_cast(V)) F->eraseFromParent(); else llvm_unreachable("Unexpected value to remove!"); } return true; } bool lowerBuiltins(SPIRVModule *BM, Module *M) { auto Format = BM->getBuiltinFormat(); if (Format == BuiltinFormat::Function && !lowerBuiltinVariablesToCalls(M)) return false; if (Format == BuiltinFormat::Global && !lowerBuiltinCallsToVariables(M)) return false; return true; } bool postProcessBuiltinReturningStruct(Function *F) { Module *M = F->getParent(); LLVMContext *Context = &M->getContext(); std::string Name = F->getName().str(); F->setName(Name + ".old"); SmallVector InstToRemove; for (auto *U : F->users()) { if (auto *CI = dyn_cast(U)) { IRBuilder<> Builder(CI->getParent()); Builder.SetInsertPoint(CI); SmallVector Users(CI->users()); Value *A = nullptr; StoreInst *SI = nullptr; for (auto *U : Users) { if ((SI = dyn_cast(U)) != nullptr) { A = SI->getPointerOperand(); InstToRemove.push_back(SI); break; } } if (!A) { A = Builder.CreateAlloca(F->getReturnType()); } SmallVector ArgTys; getFunctionTypeParameterTypes(F->getFunctionType(), ArgTys); ArgTys.insert(ArgTys.begin(), A->getType()); auto *NewF = getOrCreateFunction(M, Type::getVoidTy(*Context), ArgTys, Name); auto SretAttr = Attribute::get(*Context, Attribute::AttrKind::StructRet, F->getReturnType()); NewF->addParamAttr(0, SretAttr); NewF->setCallingConv(F->getCallingConv()); auto Args = getArguments(CI); Args.insert(Args.begin(), A); CallInst *NewCI = Builder.CreateCall( NewF, Args, NewF->getReturnType()->isVoidTy() ? "" : CI->getName()); NewCI->addParamAttr(0, SretAttr); NewCI->setCallingConv(CI->getCallingConv()); SmallVector UsersToReplace; for (auto *U : Users) if (U != SI) UsersToReplace.push_back(U); if (UsersToReplace.size() > 0) { auto *LI = Builder.CreateLoad(F->getReturnType(), A); for (auto *U : UsersToReplace) U->replaceUsesOfWith(CI, LI); } InstToRemove.push_back(CI); } } for (auto *Inst : InstToRemove) { Inst->dropAllReferences(); Inst->eraseFromParent(); } F->dropAllReferences(); F->eraseFromParent(); return true; } bool postProcessBuiltinWithArrayArguments(Function *F, StringRef DemangledName) { LLVM_DEBUG(dbgs() << "[postProcessOCLBuiltinWithArrayArguments] " << *F << '\n'); auto Attrs = F->getAttributes(); auto Name = F->getName(); mutateFunction( F, [=](CallInst *CI, std::vector &Args) { auto FBegin = CI->getFunction()->begin()->getFirstInsertionPt(); for (auto &I : Args) { auto *T = I->getType(); if (!T->isArrayTy()) continue; auto *Alloca = new AllocaInst(T, 0, "", &(*FBegin)); new StoreInst(I, Alloca, false, CI); auto *Zero = ConstantInt::getNullValue(Type::getInt32Ty(T->getContext())); Value *Index[] = {Zero, Zero}; I = GetElementPtrInst::CreateInBounds(T, Alloca, Index, "", CI); } return Name.str(); }, nullptr, &Attrs); return true; } bool postProcessBuiltinsReturningStruct(Module *M, bool IsCpp) { StringRef DemangledName; // postProcessBuiltinReturningStruct may remove some functions from the // module, so use make_early_inc_range for (auto &F : make_early_inc_range(M->functions())) { if (F.hasName() && F.isDeclaration()) { LLVM_DEBUG(dbgs() << "[postProcess sret] " << F << '\n'); if (F.getReturnType()->isStructTy() && oclIsBuiltin(F.getName(), DemangledName, IsCpp)) { if (!postProcessBuiltinReturningStruct(&F)) return false; } } } return true; } bool postProcessBuiltinsWithArrayArguments(Module *M, bool IsCpp) { StringRef DemangledName; // postProcessBuiltinWithArrayArguments may remove some functions from the // module, so use make_early_inc_range for (auto &F : make_early_inc_range(M->functions())) { if (F.hasName() && F.isDeclaration()) { LLVM_DEBUG(dbgs() << "[postProcess array arg] " << F << '\n'); if (hasArrayArg(&F) && oclIsBuiltin(F.getName(), DemangledName, IsCpp)) if (!postProcessBuiltinWithArrayArguments(&F, DemangledName)) return false; } } return true; } } // namespace SPIRV namespace { class SPIRVFriendlyIRMangleInfo : public BuiltinFuncMangleInfo { public: SPIRVFriendlyIRMangleInfo(spv::Op OC, ArrayRef ArgTys, ArrayRef Ops) : OC(OC), ArgTys(ArgTys), Ops(Ops) {} void init(StringRef UniqUnmangledName) override { UnmangledName = UniqUnmangledName.str(); switch (static_cast(OC)) { case OpConvertUToF: case OpUConvert: case OpSatConvertUToS: // Treat all arguments as unsigned addUnsignedArg(-1); break; case OpSubgroupShuffleINTEL: case OpSubgroupShuffleXorINTEL: addUnsignedArg(1); break; case OpSubgroupShuffleDownINTEL: case OpSubgroupShuffleUpINTEL: addUnsignedArg(2); break; case OpSubgroupBlockWriteINTEL: addUnsignedArg(0); addUnsignedArg(1); break; case OpSubgroupImageBlockWriteINTEL: addUnsignedArg(2); break; case OpSubgroupBlockReadINTEL: setArgAttr(0, SPIR::ATTR_CONST); addUnsignedArg(0); break; case OpAtomicUMax: case OpAtomicUMin: addUnsignedArg(0); addUnsignedArg(3); break; case OpGroupUMax: case OpGroupUMin: case OpGroupNonUniformBroadcast: case OpGroupNonUniformBallotBitCount: case OpGroupNonUniformShuffle: case OpGroupNonUniformShuffleXor: case OpGroupNonUniformShuffleUp: case OpGroupNonUniformShuffleDown: addUnsignedArg(2); break; case OpGroupNonUniformRotateKHR: if (ArgTys.size() == 4) addUnsignedArg(3); break; case OpGroupNonUniformInverseBallot: case OpGroupNonUniformBallotFindLSB: case OpGroupNonUniformBallotFindMSB: addUnsignedArg(1); break; case OpBitFieldSExtract: case OpGroupNonUniformBallotBitExtract: addUnsignedArg(1); addUnsignedArg(2); break; case OpGroupNonUniformIAdd: case OpGroupNonUniformFAdd: case OpGroupNonUniformIMul: case OpGroupNonUniformFMul: case OpGroupNonUniformSMin: case OpGroupNonUniformFMin: case OpGroupNonUniformSMax: case OpGroupNonUniformFMax: case OpGroupNonUniformBitwiseAnd: case OpGroupNonUniformBitwiseOr: case OpGroupNonUniformBitwiseXor: case OpGroupNonUniformLogicalAnd: case OpGroupNonUniformLogicalOr: case OpGroupNonUniformLogicalXor: addUnsignedArg(3); break; case OpBitFieldInsert: case OpGroupNonUniformUMax: case OpGroupNonUniformUMin: addUnsignedArg(2); addUnsignedArg(3); break; case OpEnqueueMarker: addUnsignedArg(1); break; case OpSubgroupAvcBmeInitializeINTEL: addUnsignedArgs(0, 7); break; case OpSubgroupAvcFmeInitializeINTEL: case OpSubgroupAvcSicConfigureIpeLumaINTEL: addUnsignedArgs(0, 6); break; case OpSubgroupAvcImeAdjustRefOffsetINTEL: addUnsignedArgs(1, 3); break; case OpSubgroupAvcImeGetBorderReachedINTEL: case OpSubgroupAvcImeRefWindowSizeINTEL: case OpSubgroupAvcImeSetEarlySearchTerminationThresholdINTEL: case OpSubgroupAvcImeSetMaxMotionVectorCountINTEL: case OpSubgroupAvcImeSetWeightedSadINTEL: case OpSubgroupAvcMceSetInterBaseMultiReferencePenaltyINTEL: case OpSubgroupAvcMceSetInterDirectionPenaltyINTEL: case OpSubgroupAvcMceSetInterShapePenaltyINTEL: case OpSubgroupAvcMceSetSingleReferenceInterlacedFieldPolarityINTEL: case OpSubgroupAvcMceSetSourceInterlacedFieldPolarityINTEL: case OpSubgroupAvcSicInitializeINTEL: case OpSubgroupAvcSicSetBlockBasedRawSkipSadINTEL: case OpSubgroupAvcSicSetIntraChromaModeCostFunctionINTEL: case OpSubgroupAvcSicSetIntraLumaShapePenaltyINTEL: case OpSubgroupAvcSicSetSkcForwardTransformEnableINTEL: addUnsignedArg(0); break; case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeDistortionsINTEL: case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeMotionVectorsINTEL: case OpSubgroupAvcImeGetStreamoutDualReferenceMajorShapeReferenceIdsINTEL: case OpSubgroupAvcRefEvaluateWithMultiReferenceInterlacedINTEL: case OpSubgroupAvcSicEvaluateWithMultiReferenceInterlacedINTEL: case OpSubgroupAvcRefEvaluateWithMultiReferenceINTEL: case OpSubgroupAvcSicEvaluateWithMultiReferenceINTEL: addUnsignedArgs(1, 2); break; case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeDistortionsINTEL: case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeMotionVectorsINTEL: case OpSubgroupAvcImeGetStreamoutSingleReferenceMajorShapeReferenceIdsINTEL: case OpSubgroupAvcImeSetSingleReferenceINTEL: addUnsignedArg(1); break; case OpBitFieldUExtract: case OpSubgroupAvcImeInitializeINTEL: case OpSubgroupAvcMceSetMotionVectorCostFunctionINTEL: case OpSubgroupAvcSicSetIntraLumaModeCostFunctionINTEL: addUnsignedArgs(0, 2); break; case OpSubgroupAvcImeSetDualReferenceINTEL: addUnsignedArg(2); break; case OpSubgroupAvcMceGetDefaultInterBaseMultiReferencePenaltyINTEL: case OpSubgroupAvcMceGetDefaultInterDirectionPenaltyINTEL: case OpSubgroupAvcMceGetDefaultInterMotionVectorCostTableINTEL: case OpSubgroupAvcMceGetDefaultInterShapePenaltyINTEL: case OpSubgroupAvcMceGetDefaultIntraLumaModePenaltyINTEL: case OpSubgroupAvcMceGetDefaultIntraLumaShapePenaltyINTEL: case OpSubgroupAvcMceGetInterReferenceInterlacedFieldPolaritiesINTEL: case OpSubgroupAvcMceSetDualReferenceInterlacedFieldPolaritiesINTEL: case OpSubgroupAvcSicGetMotionVectorMaskINTEL: addUnsignedArgs(0, 1); break; case OpSubgroupAvcSicConfigureIpeLumaChromaINTEL: addUnsignedArgs(0, 9); break; case OpSubgroupAvcSicConfigureSkcINTEL: addUnsignedArgs(0, 4); break; case OpUDotKHR: case OpUDotAccSatKHR: addUnsignedArg(-1); break; case OpSUDotKHR: case OpSUDotAccSatKHR: addUnsignedArg(1); break; case OpImageWrite: { const size_t Idx = getImageOperandsIndex(OC); if (Ops.size() > Idx) { auto ImOp = static_cast(Ops[Idx])->getZExtIntValue(); if (ImOp & ImageOperandsMask::ImageOperandsZeroExtendMask) addUnsignedArg(2); } break; } case internal::OpConvertHandleToImageINTEL: case internal::OpConvertHandleToSamplerINTEL: case internal::OpConvertHandleToSampledImageINTEL: addUnsignedArg(0); break; default:; // No special handling is needed } } private: spv::Op OC; ArrayRef ArgTys; ArrayRef Ops; }; class OpenCLStdToSPIRVFriendlyIRMangleInfo : public BuiltinFuncMangleInfo { public: OpenCLStdToSPIRVFriendlyIRMangleInfo(OCLExtOpKind ExtOpId, ArrayRef ArgTys, Type *RetTy) : ExtOpId(ExtOpId), ArgTys(ArgTys) { std::string Postfix = ""; if (needRetTypePostfix()) Postfix = kSPIRVPostfix::Divider + getPostfixForReturnType(RetTy, true); UnmangledName = getSPIRVExtFuncName(SPIRVEIS_OpenCL, ExtOpId, Postfix); } bool needRetTypePostfix() { switch (ExtOpId) { case OpenCLLIB::Vload_half: case OpenCLLIB::Vload_halfn: case OpenCLLIB::Vloada_halfn: case OpenCLLIB::Vloadn: return true; default: return false; } } void init(StringRef) override { switch (ExtOpId) { case OpenCLLIB::UAbs: case OpenCLLIB::UAbs_diff: case OpenCLLIB::UAdd_sat: case OpenCLLIB::UHadd: case OpenCLLIB::URhadd: case OpenCLLIB::UClamp: case OpenCLLIB::UMad_hi: case OpenCLLIB::UMad_sat: case OpenCLLIB::UMax: case OpenCLLIB::UMin: case OpenCLLIB::UMul_hi: case OpenCLLIB::USub_sat: case OpenCLLIB::U_Upsample: case OpenCLLIB::UMad24: case OpenCLLIB::UMul24: // Treat all arguments as unsigned addUnsignedArg(-1); break; case OpenCLLIB::S_Upsample: addUnsignedArg(1); break; default:; // No special handling is needed } } private: OCLExtOpKind ExtOpId; ArrayRef ArgTys; }; } // namespace namespace SPIRV { void BuiltinFuncMangleInfo::fillPointerElementTypes( ArrayRef PointerElementTys) { for (unsigned I = 0; I < PointerElementTys.size(); I++) { getTypeMangleInfo(I).PointerElementType = PointerElementTys[I]; } } std::string getSPIRVFriendlyIRFunctionName(OCLExtOpKind ExtOpId, ArrayRef ArgTys, ArrayRef PointerElementTys, Type *RetTy) { OpenCLStdToSPIRVFriendlyIRMangleInfo MangleInfo(ExtOpId, ArgTys, RetTy); MangleInfo.fillPointerElementTypes(PointerElementTys); return mangleBuiltin(MangleInfo.getUnmangledName(), ArgTys, &MangleInfo); } std::string getSPIRVFriendlyIRFunctionName(const std::string &UniqName, spv::Op OC, ArrayRef ArgTys, ArrayRef PointerElementTys, ArrayRef Ops) { SPIRVFriendlyIRMangleInfo MangleInfo(OC, ArgTys, Ops); MangleInfo.fillPointerElementTypes(PointerElementTys); return mangleBuiltin(UniqName, ArgTys, &MangleInfo); } template MetadataAsValue *map2MDString(LLVMContext &C, SPIRVValue *V) { if (V->getOpCode() != OpConstant) return nullptr; uint64_t Const = static_cast(V)->getZExtIntValue(); std::string Str = SPIRVMap::map(static_cast(Const)); return MetadataAsValue::get(C, MDString::get(C, Str)); } template MetadataAsValue * map2MDString(LLVMContext &, SPIRVValue *); template MetadataAsValue *map2MDString(LLVMContext &, SPIRVValue *); [[nodiscard]] SPIRVWord bitCeil(SPIRVWord Value) { if (Value < 2) return 1; // If Value is already a power of 2, just return it. if ((Value & (Value - 1)) == 0) return Value; Value--; for (SPIRVWord Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { Value |= Value >> Shift; } return ++Value; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVWriter.cpp000066400000000000000000007323111506646577400217770ustar00rootroot00000000000000//===- SPIRVWriter.cpp - Converts LLVM to SPIR-V ----------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements conversion of LLVM intermediate language to SPIR-V /// binary. /// //===----------------------------------------------------------------------===// #include "SPIRVWriter.h" #include "LLVMToSPIRVDbgTran.h" #include "OCLToSPIRV.h" #include "PreprocessMetadata.h" #include "SPIRVAsm.h" #include "SPIRVBasicBlock.h" #include "SPIRVEntry.h" #include "SPIRVEnum.h" #include "SPIRVExtInst.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVInternal.h" #include "SPIRVLLVMUtil.h" #include "SPIRVLowerBitCastToNonStandardType.h" #include "SPIRVLowerBool.h" #include "SPIRVLowerConstExpr.h" #include "SPIRVLowerMemmove.h" #include "SPIRVLowerOCLBlocks.h" #include "SPIRVLowerSaddIntrinsics.h" #include "SPIRVMDWalker.h" #include "SPIRVMemAliasingINTEL.h" #include "SPIRVModule.h" #include "SPIRVRegularizeLLVM.h" #include "SPIRVType.h" #include "SPIRVUtil.h" #include "SPIRVValue.h" #include "VectorComputeUtil.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/Pass.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/Mem2Reg.h" #include #include #include #include #include #include #include #include #define DEBUG_TYPE "spirv" using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace { static SPIRVWord convertFloatToSPIRVWord(float F) { union { float F; SPIRVWord Spir; } FPMaxError; FPMaxError.F = F; return FPMaxError.Spir; } } // namespace namespace SPIRV { static void foreachKernelArgMD( MDNode *MD, SPIRVFunction *BF, std::function Func) { for (unsigned I = 0, E = MD->getNumOperands(); I != E; ++I) { SPIRVFunctionParameter *BA = BF->getArgument(I); Func(getMDOperandAsString(MD, I).str(), BA); } } static void foreachKernelArgMD( MDNode *MD, SPIRVFunction *BF, std::function Func) { for (unsigned I = 0, E = MD->getNumOperands(); I != E; ++I) { SPIRVFunctionParameter *BA = BF->getArgument(I); Func(MD->getOperand(I), BA); } } static SPIRVMemoryModelKind getMemoryModel(Module &M) { auto *MemoryModelMD = M.getNamedMetadata(kSPIRVMD::MemoryModel); if (MemoryModelMD && (MemoryModelMD->getNumOperands() > 0)) { auto *Ref0 = MemoryModelMD->getOperand(0); if (Ref0 && Ref0->getNumOperands() > 1) { auto &&ModelOp = Ref0->getOperand(1); auto *ModelCI = mdconst::dyn_extract(ModelOp); if (ModelCI && (ModelCI->getValue().getActiveBits() <= 64)) { auto Model = static_cast(ModelCI->getZExtValue()); return Model; } } } return SPIRVMemoryModelKind::MemoryModelMax; } static void translateSEVDecoration(Attribute Sev, SPIRVValue *Val) { assert(Sev.isStringAttribute() && Sev.getKindAsString() == kVCMetadata::VCSingleElementVector); auto *Ty = Val->getType(); assert((Ty->isTypeBool() || Ty->isTypeFloat() || Ty->isTypeInt() || Ty->isTypePointer()) && "This decoration is valid only for Scalar or Pointer types"); if (Ty->isTypePointer()) { SPIRVWord IndirectLevelsOnElement = 0; Sev.getValueAsString().getAsInteger(0, IndirectLevelsOnElement); Val->addDecorate(DecorationSingleElementVectorINTEL, IndirectLevelsOnElement); } else Val->addDecorate(DecorationSingleElementVectorINTEL); } LLVMToSPIRVBase::LLVMToSPIRVBase(SPIRVModule *SMod) : M(nullptr), Ctx(nullptr), BM(SMod), SrcLang(0), SrcLangVer(0) { DbgTran = std::make_unique(nullptr, SMod, this); } LLVMToSPIRVBase::~LLVMToSPIRVBase() { for (auto *I : UnboundInst) I->deleteValue(); } bool LLVMToSPIRVBase::runLLVMToSPIRV(Module &Mod) { M = &Mod; CG = std::make_unique(Mod); Ctx = &M->getContext(); DbgTran->setModule(M); assert(BM && "SPIR-V module not initialized"); translate(); return true; } SPIRVValue *LLVMToSPIRVBase::getTranslatedValue(const Value *V) const { auto Loc = ValueMap.find(V); if (Loc != ValueMap.end()) return Loc->second; return nullptr; } bool LLVMToSPIRVBase::isKernel(Function *F) { if (F->getCallingConv() == CallingConv::SPIR_KERNEL) return true; return false; } bool LLVMToSPIRVBase::isBuiltinTransToInst(Function *F) { StringRef DemangledName; if (!oclIsBuiltin(F->getName(), DemangledName) && !isDecoratedSPIRVFunc(F, DemangledName)) return false; SPIRVDBG(spvdbgs() << "CallInst: demangled name: " << DemangledName.str() << '\n'); return getSPIRVFuncOC(DemangledName) != OpNop; } bool LLVMToSPIRVBase::isBuiltinTransToExtInst( Function *F, SPIRVExtInstSetKind *ExtSet, SPIRVWord *ExtOp, SmallVectorImpl *Dec) { StringRef DemangledName; if (!oclIsBuiltin(F->getName(), DemangledName)) return false; LLVM_DEBUG(dbgs() << "[oclIsBuiltinTransToExtInst] CallInst: demangled name: " << DemangledName << '\n'); StringRef S = DemangledName; if (!S.startswith(kSPIRVName::Prefix)) return false; S = S.drop_front(strlen(kSPIRVName::Prefix)); auto Loc = S.find(kSPIRVPostfix::Divider); auto ExtSetName = S.substr(0, Loc); SPIRVExtInstSetKind Set = SPIRVEIS_Count; if (!SPIRVExtSetShortNameMap::rfind(ExtSetName.str(), &Set)) return false; assert((Set == SPIRVEIS_OpenCL || Set == BM->getDebugInfoEIS()) && "Unsupported extended instruction set"); auto ExtOpName = S.substr(Loc + 1); auto Splited = ExtOpName.split(kSPIRVPostfix::ExtDivider); OCLExtOpKind EOC; if (!OCLExtOpMap::rfind(Splited.first.str(), &EOC)) return false; if (ExtSet) *ExtSet = Set; if (ExtOp) *ExtOp = EOC; if (Dec) { SmallVector P; Splited.second.split(P, kSPIRVPostfix::Divider); for (auto &I : P) Dec->push_back(I.str()); } return true; } bool isUniformGroupOperation(Function *F) { auto Name = F->getName(); if (Name.contains("GroupIMulKHR") || Name.contains("GroupFMulKHR") || Name.contains("GroupBitwiseAndKHR") || Name.contains("GroupBitwiseOrKHR") || Name.contains("GroupBitwiseXorKHR") || Name.contains("GroupLogicalAndKHR") || Name.contains("GroupLogicalOrKHR") || Name.contains("GroupLogicalXorKHR")) return true; return false; } static bool recursiveType(const StructType *ST, const Type *Ty) { SmallPtrSet Seen; std::function Run = [&](const Type *Ty) { if (auto *StructTy = dyn_cast(Ty)) { if (StructTy == ST) return true; if (Seen.count(StructTy)) return false; Seen.insert(StructTy); return find_if(StructTy->element_begin(), StructTy->element_end(), Run) != StructTy->element_end(); } // Opaque pointers are translated to i8*, so they're not going to create // recursive types. if (Ty->isPointerTy() && !Ty->isOpaquePointerTy()) { Type *ElTy = Ty->getNonOpaquePointerElementType(); if (auto *FTy = dyn_cast(ElTy)) { // If we have a function pointer, then argument types and return type of // the referenced function also need to be checked return Run(FTy->getReturnType()) || any_of(FTy->param_begin(), FTy->param_end(), Run); } return Run(ElTy); } if (auto *ArrayTy = dyn_cast(Ty)) return Run(ArrayTy->getArrayElementType()); return false; }; return Run(Ty); } SPIRVType *LLVMToSPIRVBase::transType(Type *T) { LLVMToSPIRVTypeMap::iterator Loc = TypeMap.find(T); if (Loc != TypeMap.end()) return Loc->second; SPIRVDBG(dbgs() << "[transType] " << *T << '\n'); if (T->isVoidTy()) return mapType(T, BM->addVoidType()); if (T->isIntegerTy(1)) return mapType(T, BM->addBoolType()); if (T->isIntegerTy()) { unsigned BitWidth = T->getIntegerBitWidth(); // SPIR-V 2.16.1. Universal Validation Rules: Scalar integer types can be // parameterized only as 32 bit, plus any additional sizes enabled by // capabilities. if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_integers) || BM->getErrorLog().checkError( (BitWidth == 4 && BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_int4)) || BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64, SPIRVEC_InvalidBitWidth, std::to_string(BitWidth))) { return mapType(T, BM->addIntegerType(T->getIntegerBitWidth())); } } if (T->isBFloatTy()) { BM->getErrorLog().checkError( BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_bfloat16), SPIRVEC_RequiresExtension, "SPV_KHR_bfloat16\n" "NOTE: LLVM module contains bfloat type, translation of which " "requires this extension"); return mapType(T, BM->addFloatType(16, FPEncodingBFloat16KHR)); } if (T->isFloatingPointTy()) return mapType(T, BM->addFloatType(T->getPrimitiveSizeInBits())); // A pointer to image or pipe type in LLVM is translated to a SPIRV // (non-pointer) image or pipe type. if (T->isPointerTy()) { auto *ET = T->isOpaquePointerTy() ? Type::getInt8Ty(T->getContext()) : T->getNonOpaquePointerElementType(); auto AddrSpc = T->getPointerAddressSpace(); return transPointerType(ET, AddrSpc); } if (auto *VecTy = dyn_cast(T)) { if (VecTy->getElementType()->isPointerTy()) { // SPV_INTEL_masked_gather_scatter extension changes 2.16.1. Universal // Validation Rules: // Vector types must be parameterized only with numerical types, // [Physical Pointer Type] types or the [OpTypeBool] type. // Without it vector of pointers is not allowed in SPIR-V. if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_masked_gather_scatter)) { BM->getErrorLog().checkError( false, SPIRVEC_RequiresExtension, "SPV_INTEL_masked_gather_scatter\n" "NOTE: LLVM module contains vector of pointers, translation " "of which requires this extension"); return nullptr; } BM->addExtension(ExtensionID::SPV_INTEL_masked_gather_scatter); BM->addCapability(internal::CapabilityMaskedGatherScatterINTEL); } return mapType(T, BM->addVectorType(transType(VecTy->getElementType()), VecTy->getNumElements())); } if (T->isArrayTy()) { // SPIR-V 1.3 s3.32.6: Length is the number of elements in the array. // It must be at least 1. if (T->getArrayNumElements() < 1) { std::string Str; llvm::raw_string_ostream OS(Str); OS << *T; SPIRVCK(T->getArrayNumElements() >= 1, InvalidArraySize, OS.str()); } return mapType(T, BM->addArrayType( transType(T->getArrayElementType()), static_cast(transValue( ConstantInt::get(getSizetType(), T->getArrayNumElements(), false), nullptr)))); } if (T->isStructTy() && !T->isSized()) { auto ST = dyn_cast(T); (void)ST; // Silence warning assert(!ST->getName().startswith(kSPR2TypeName::PipeRO)); assert(!ST->getName().startswith(kSPR2TypeName::PipeWO)); assert(!ST->getName().startswith(kSPR2TypeName::ImagePrefix)); return mapType(T, BM->addOpaqueType(T->getStructName().str())); } if (auto ST = dyn_cast(T)) { assert(ST->isSized()); StringRef Name; if (ST->hasName()) Name = ST->getName(); if (Name == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler)) return transSPIRVOpaqueType(getSPIRVTypeName(kSPIRVTypeName::Sampler), SPIRAS_Constant); if (Name == getSPIRVTypeName(kSPIRVTypeName::ConstantPipeStorage)) return transSPIRVOpaqueType(getSPIRVTypeName(kSPIRVTypeName::PipeStorage), SPIRAS_Constant); constexpr size_t MaxNumElements = MaxWordCount - SPIRVTypeStruct::FixedWC; const size_t NumElements = ST->getNumElements(); size_t SPIRVStructNumElements = NumElements; // In case number of elements is greater than maximum WordCount and // SPV_INTEL_long_constant_composite is not enabled, the error will be // emitted by validate functionality of SPIRVTypeStruct class. if (NumElements > MaxNumElements && BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_long_constant_composite)) { SPIRVStructNumElements = MaxNumElements; } auto *Struct = BM->openStructType(SPIRVStructNumElements, Name.str()); mapType(T, Struct); if (NumElements > MaxNumElements && BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_long_constant_composite)) { uint64_t NumOfContinuedInstructions = NumElements / MaxNumElements - 1; for (uint64_t J = 0; J < NumOfContinuedInstructions; J++) { auto *Continued = BM->addTypeStructContinuedINTEL(MaxNumElements); Struct->addContinuedInstruction( static_cast(Continued)); } uint64_t Remains = NumElements % MaxNumElements; if (Remains) { auto *Continued = BM->addTypeStructContinuedINTEL(Remains); Struct->addContinuedInstruction( static_cast(Continued)); } } SmallVector ForwardRefs; for (unsigned I = 0, E = T->getStructNumElements(); I != E; ++I) { auto *ElemTy = ST->getElementType(I); if ((isa(ElemTy) || isa(ElemTy) || isa(ElemTy) || isa(ElemTy)) && recursiveType(ST, ElemTy)) ForwardRefs.push_back(I); else Struct->setMemberType(I, transType(ST->getElementType(I))); } BM->closeStructType(Struct, ST->isPacked()); for (auto I : ForwardRefs) Struct->setMemberType(I, transType(ST->getElementType(I))); return Struct; } if (FunctionType *FT = dyn_cast(T)) { SPIRVType *RT = transType(FT->getReturnType()); std::vector PT; for (FunctionType::param_iterator I = FT->param_begin(), E = FT->param_end(); I != E; ++I) PT.push_back(transType(*I)); return mapType(T, getSPIRVFunctionType(RT, PT)); } if (T->isTokenTy()) { BM->getErrorLog().checkError( false, SPIRVEC_InvalidModule, "LLVM module contains token type, which doesn't have a counterpart in " "SPIR-V"); return nullptr; } llvm_unreachable("Not implemented!"); return 0; } SPIRVType *LLVMToSPIRVBase::transPointerType(Type *ET, unsigned AddrSpc) { Type *T = PointerType::get(ET, AddrSpc); if (ET->isFunctionTy() && !BM->checkExtension(ExtensionID::SPV_INTEL_function_pointers, SPIRVEC_FunctionPointers, toString(T))) return nullptr; std::string TypeKey = (Twine((uintptr_t)ET) + Twine(AddrSpc)).str(); auto Loc = PointeeTypeMap.find(TypeKey); if (Loc != PointeeTypeMap.end()) return Loc->second; // A pointer to image or pipe type in LLVM is translated to a SPIRV // (non-pointer) image or pipe type. auto *ST = dyn_cast(ET); // Lower global_device and global_host address spaces that were added in // SYCL as part of SYCL_INTEL_usm_address_spaces extension to just global // address space if device doesn't support SPV_INTEL_usm_storage_classes // extension if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_usm_storage_classes) && ((AddrSpc == SPIRAS_GlobalDevice) || (AddrSpc == SPIRAS_GlobalHost))) { return transPointerType(ET, SPIRAS_Global); } // Lower function pointer address space to private if // spirv-emit-function-ptr-addr-space is not passed if (AddrSpc == SPIRAS_CodeSectionINTEL && !BM->shouldEmitFunctionPtrAddrSpace()) return transPointerType(ET, SPIRAS_Private); if (ST && !ST->isSized()) { Op OpCode; StringRef STName = ST->getName(); // Workaround for non-conformant SPIR binary if (STName == "struct._event_t") { STName = kSPR2TypeName::Event; ST->setName(STName); } std::pair Key = {STName, AddrSpc}; if (auto *MappedTy = OpaqueStructMap.lookup(Key)) return MappedTy; auto SaveType = [&](SPIRVType *MappedTy) { OpaqueStructMap[Key] = MappedTy; PointeeTypeMap[TypeKey] = MappedTy; return MappedTy; }; if (STName.startswith(kSPR2TypeName::PipeRO) || STName.startswith(kSPR2TypeName::PipeWO)) { auto *PipeT = BM->addPipeType(); PipeT->setPipeAcessQualifier(STName.startswith(kSPR2TypeName::PipeRO) ? AccessQualifierReadOnly : AccessQualifierWriteOnly); return SaveType(PipeT); } if (STName.startswith(kSPR2TypeName::ImagePrefix)) { assert(AddrSpc == SPIRAS_Global); Type *ImageTy = adaptSPIRVImageType(M, ST); return SaveType(transPointerType(ImageTy, SPIRAS_Global)); } if (STName == kSPR2TypeName::Sampler) return SaveType(transSPIRVOpaqueType( getSPIRVTypeName(kSPIRVTypeName::Sampler), SPIRAS_Constant)); if (STName.startswith(kSPIRVTypeName::PrefixAndDelim)) return transSPIRVOpaqueType(STName, AddrSpc); if (STName.startswith(kOCLSubgroupsAVCIntel::TypePrefix)) return SaveType(BM->addSubgroupAvcINTELType( OCLSubgroupINTELTypeOpCodeMap::map(ST->getName().str()))); if (OCLOpaqueTypeOpCodeMap::find(STName.str(), &OpCode)) { switch (OpCode) { default: return SaveType(BM->addOpaqueGenericType(OpCode)); case OpTypeDeviceEvent: return SaveType(BM->addDeviceEventType()); case OpTypeQueue: return SaveType(BM->addQueueType()); } } if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) { if (STName.startswith(kVCType::VCBufferSurface)) { // VCBufferSurface always have Access Qualifier auto Access = getAccessQualifier(STName); return SaveType(BM->addBufferSurfaceINTELType(Access)); } } if (ST->isOpaque()) { return SaveType(BM->addPointerType( SPIRSPIRVAddrSpaceMap::map(static_cast(AddrSpc)), transType(ET))); } } else { // JointMatrixINTEL type is not necessarily an opaque type, it can be // represented as a structure with pointer to a multidimensional array // member. if (ST && ST->hasName()) { StringRef STName = ST->getName(); if (STName.startswith(kSPIRVTypeName::PrefixAndDelim)) { SmallVector Postfixes; auto TN = decodeSPIRVTypeName(STName, Postfixes); if (TN == kSPIRVTypeName::JointMatrixINTEL || TN == kSPIRVTypeName::CooperativeMatrixKHR) { SPIRVType *TranslatedTy = transSPIRVJointOrCooperativeMatrixType( Postfixes, TN == kSPIRVTypeName::CooperativeMatrixKHR); PointeeTypeMap[TypeKey] = TranslatedTy; return TranslatedTy; } } } SPIRVType *ElementType = transType(ET); // ET, as a recursive type, may contain exactly the same pointer T, so it // may happen that after translation of ET we already have translated T, // added the translated pointer to the SPIR-V module and mapped T to this // pointer. Now we have to check PointeeTypeMap again. auto Loc = PointeeTypeMap.find(TypeKey); if (Loc != PointeeTypeMap.end()) { return Loc->second; } SPIRVType *TranslatedTy = transPointerType(ElementType, AddrSpc); PointeeTypeMap[TypeKey] = TranslatedTy; return TranslatedTy; } llvm_unreachable("Not implemented!"); return nullptr; } SPIRVType *LLVMToSPIRVBase::transPointerType(SPIRVType *ET, unsigned AddrSpc) { std::string TypeKey = (Twine((uintptr_t)ET) + Twine(AddrSpc)).str(); auto Loc = PointeeTypeMap.find(TypeKey); if (Loc != PointeeTypeMap.end()) return Loc->second; if (AddrSpc == SPIRAS_CodeSectionINTEL && !BM->shouldEmitFunctionPtrAddrSpace()) return transPointerType(ET, SPIRAS_Private); SPIRVType *TranslatedTy = BM->addPointerType( SPIRSPIRVAddrSpaceMap::map(static_cast(AddrSpc)), ET); PointeeTypeMap[TypeKey] = TranslatedTy; return TranslatedTy; } // Representation in LLVM IR before the translator is a pointer to an opaque // structure: // %spirv.JointMatrixINTEL._%element_type%_%rows%_%cols%_%scope%_%use% // Here we check the structure name yet again. Another option would be to // check SPIR-V friendly function calls (by their name) and obtain return // or their parameter types, assuming, that the appropriate types are Matrix // structure type. But in the near future, we will reuse Composite // instructions to do, for example, matrix initialization directly on AMX // register by OpCompositeConstruct. And we can't claim, that the Result type // of OpCompositeConstruct instruction is always the joint matrix type, it's // simply not true. SPIRVType *LLVMToSPIRVBase::transSPIRVJointOrCooperativeMatrixType( SmallVector Postfixes, bool IsCooperative) { Type *ElemTy = nullptr; StringRef Ty{Postfixes[0]}; auto NumBits = llvm::StringSwitch(Ty) .Case("char", 8) .Case("short", 16) .Case("int", 32) .Case("long", 64) .Default(0); if (NumBits) ElemTy = IntegerType::get(M->getContext(), NumBits); else if (Ty == "half") ElemTy = Type::getHalfTy(M->getContext()); else if (Ty == "float") ElemTy = Type::getFloatTy(M->getContext()); else if (Ty == "double") ElemTy = Type::getDoubleTy(M->getContext()); else if (Ty == "bfloat16") ElemTy = BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_bfloat16) ? Type::getBFloatTy(M->getContext()) : Type::getInt16Ty(M->getContext()); else llvm_unreachable("Unexpected type for matrix!"); auto ParseInteger = [this](StringRef Postfix) -> ConstantInt * { unsigned long long N = 0; consumeUnsignedInteger(Postfix, 10, N); return getUInt32(M, N); }; std::vector Args; for (size_t I = 1; I != Postfixes.size(); ++I) Args.emplace_back(transConstant(ParseInteger(Postfixes[I]))); if (IsCooperative) return BM->addCooperativeMatrixKHRType(transType(ElemTy), Args); return BM->addJointMatrixINTELType(transType(ElemTy), Args); } SPIRVType *LLVMToSPIRVBase::transSPIRVOpaqueType(StringRef STName, unsigned AddrSpace) { std::pair Key = {STName, AddrSpace}; if (auto *MappedTy = OpaqueStructMap.lookup(Key)) return MappedTy; auto SaveType = [&](SPIRVType *MappedTy) { OpaqueStructMap[Key] = MappedTy; return MappedTy; }; StructType *ST = StructType::getTypeByName(M->getContext(), STName); assert(STName.startswith(kSPIRVTypeName::PrefixAndDelim) && "Invalid SPIR-V opaque type name"); SmallVector Postfixes; auto TN = decodeSPIRVTypeName(STName, Postfixes); if (TN == kSPIRVTypeName::Pipe) { assert(AddrSpace == SPIRAS_Global); assert(Postfixes.size() == 1 && "Invalid pipe type ops"); auto PipeT = BM->addPipeType(); PipeT->setPipeAcessQualifier( static_cast(atoi(Postfixes[0].c_str()))); return SaveType(PipeT); } else if (TN == kSPIRVTypeName::Image) { assert(AddrSpace == SPIRAS_Global); // The sampled type needs to be translated through LLVM type to guarantee // uniqueness. auto SampledT = transType( getLLVMTypeForSPIRVImageSampledTypePostfix(Postfixes[0], *Ctx)); SmallVector Ops; for (unsigned I = 1; I < 8; ++I) Ops.push_back(atoi(Postfixes[I].c_str())); SPIRVTypeImageDescriptor Desc(static_cast(Ops[0]), Ops[1], Ops[2], Ops[3], Ops[4], Ops[5]); return SaveType(BM->addImageType( SampledT, Desc, static_cast(Ops[6]))); } else if (TN == kSPIRVTypeName::SampledImg) { return SaveType( BM->addSampledImageType(static_cast(transPointerType( getSPIRVStructTypeByChangeBaseTypeName( M, ST, kSPIRVTypeName::SampledImg, kSPIRVTypeName::Image), SPIRAS_Global)))); } else if (TN == kSPIRVTypeName::VmeImageINTEL) { // This type is the same as SampledImageType, but consumed by Subgroup AVC // Intel extension instructions. return SaveType( BM->addVmeImageINTELType(static_cast(transPointerType( getSPIRVStructTypeByChangeBaseTypeName( M, ST, kSPIRVTypeName::VmeImageINTEL, kSPIRVTypeName::Image), SPIRAS_Global)))); } else if (TN == kSPIRVTypeName::Sampler) return SaveType(BM->addSamplerType()); else if (TN == kSPIRVTypeName::DeviceEvent) return SaveType(BM->addDeviceEventType()); else if (TN == kSPIRVTypeName::Queue) return SaveType(BM->addQueueType()); else if (TN == kSPIRVTypeName::PipeStorage) return SaveType(BM->addPipeStorageType()); else if (TN == kSPIRVTypeName::JointMatrixINTEL || TN == kSPIRVTypeName::CooperativeMatrixKHR) { return SaveType(transSPIRVJointOrCooperativeMatrixType( Postfixes, TN == kSPIRVTypeName::CooperativeMatrixKHR)); } else return SaveType( BM->addOpaqueGenericType(SPIRVOpaqueTypeOpCodeMap::map(TN))); } SPIRVType *LLVMToSPIRVBase::transScavengedType(Value *V) { Type *Ty = V->getType(); if (!Ty->isPointerTy()) return transType(Ty); if (auto *F = dyn_cast(V)) { SPIRVType *RT = transType(F->getReturnType()); std::vector PT; for (Argument &Arg : F->args()) { auto TypePair = OCLTypeToSPIRVPtr->getAdaptedArgumentType(F, Arg.getArgNo()); Type *Ty = TypePair.first; Type *PointeeTy = TypePair.second; if (!Ty) { Ty = Arg.getType(); if (Ty->isPointerTy()) PointeeTy = Scavenger->getArgumentPointerElementType(F, Arg.getArgNo()); } SPIRVType *TransTy = nullptr; if (Ty->isPointerTy()) TransTy = transPointerType(PointeeTy, Ty->getPointerAddressSpace()); else TransTy = transType(Ty); PT.push_back(TransTy); } return getSPIRVFunctionType(RT, PT); } auto PointeeTy = Scavenger->getPointerElementType(V); auto AddrSpace = Ty->getPointerAddressSpace(); if (auto *AsTy = dyn_cast(PointeeTy)) return transPointerType(AsTy, AddrSpace); return transPointerType(transScavengedType(cast(PointeeTy)), AddrSpace); } SPIRVType * LLVMToSPIRVBase::getSPIRVFunctionType(SPIRVType *RT, const std::vector &Args) { // Come up with a unique string identifier for the arguments. This is a hacky // way of doing so, but it works. std::string TypeKey; llvm::raw_string_ostream TKS(TypeKey); TKS << (uintptr_t)RT << ","; for (SPIRVType *ArgTy : Args) { TKS << (uintptr_t)ArgTy << ","; } // Create a SPIRVType for the function type. Since SPIRVModule doesn't do // any type uniquing for SPIRVType, we have to do it ourself. TKS.flush(); auto It = PointeeTypeMap.find(TypeKey); if (It == PointeeTypeMap.end()) It = PointeeTypeMap.insert({TypeKey, BM->addFunctionType(RT, Args)}).first; return It->second; } SPIRVFunction *LLVMToSPIRVBase::transFunctionDecl(Function *F) { if (auto BF = getTranslatedValue(F)) return static_cast(BF); if (F->isIntrinsic() && (!BM->isSPIRVAllowUnknownIntrinsicsEnabled() || isKnownIntrinsic(F->getIntrinsicID()))) { // We should not translate LLVM intrinsics as a function assert(none_of(F->users(), [this](User *U) { return getTranslatedValue(U); }) && "LLVM intrinsics shouldn't be called in SPIRV"); return nullptr; } SPIRVTypeFunction *BFT = static_cast(transScavengedType(F)); SPIRVFunction *BF = static_cast(mapValue(F, BM->addFunction(BFT))); BF->setFunctionControlMask(transFunctionControlMask(F)); if (F->hasName()) { if (isKernel(F)) { /* strip the prefix as the runtime will be looking for this name */ std::string Prefix = kSPIRVName::EntrypointPrefix; std::string Name = F->getName().str(); BM->setName(BF, Name.substr(Prefix.size())); } else { if (isUniformGroupOperation(F)) BM->getErrorLog().checkError( BM->isAllowedToUseExtension( ExtensionID::SPV_KHR_uniform_group_instructions), SPIRVEC_RequiresExtension, "SPV_KHR_uniform_group_instructions\n"); BM->setName(BF, F->getName().str()); } } if (!isKernel(F) && F->getLinkage() != GlobalValue::InternalLinkage) BF->setLinkageType(transLinkageType(F)); // Translate OpenCL/SYCL buffer_location metadata if it's attached to the // translated function declaration MDNode *BufferLocation = nullptr; if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_buffer_location)) BufferLocation = F->getMetadata("kernel_arg_buffer_location"); // Translate runtime_aligned metadata if it's attached to the translated // function declaration MDNode *RuntimeAligned = nullptr; if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_runtime_aligned)) RuntimeAligned = F->getMetadata("kernel_arg_runtime_aligned"); auto Attrs = F->getAttributes(); for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { auto ArgNo = I->getArgNo(); SPIRVFunctionParameter *BA = BF->getArgument(ArgNo); if (I->hasName()) BM->setName(BA, I->getName().str()); if (I->hasByValAttr()) BA->addAttr(FunctionParameterAttributeByVal); if (I->hasNoAliasAttr()) BA->addAttr(FunctionParameterAttributeNoAlias); if (I->hasNoCaptureAttr()) BA->addAttr(FunctionParameterAttributeNoCapture); if (I->hasStructRetAttr()) BA->addAttr(FunctionParameterAttributeSret); if (Attrs.hasParamAttr(ArgNo, Attribute::ReadOnly)) BA->addAttr(FunctionParameterAttributeNoWrite); if (Attrs.hasParamAttr(ArgNo, Attribute::ReadNone)) BA->addAttr(FunctionParameterAttributeNoReadWrite); if (Attrs.hasParamAttr(ArgNo, Attribute::ZExt)) BA->addAttr(FunctionParameterAttributeZext); if (Attrs.hasParamAttr(ArgNo, Attribute::SExt)) BA->addAttr(FunctionParameterAttributeSext); if (Attrs.hasParamAttr(ArgNo, Attribute::Alignment)) { SPIRVWord AlignmentBytes = Attrs.getParamAttr(ArgNo, Attribute::Alignment) .getAlignment() .valueOrOne() .value(); BA->setAlignment(AlignmentBytes); } if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_1) && Attrs.hasParamAttr(ArgNo, Attribute::Dereferenceable)) BA->addDecorate(DecorationMaxByteOffset, Attrs.getParamAttr(ArgNo, Attribute::Dereferenceable) .getDereferenceableBytes()); if (BufferLocation && I->getType()->isPointerTy()) { // Order of integer numbers in MD node follows the order of function // parameters on which we shall attach the appropriate decoration. Add // decoration only if MD value is not negative. int LocID = -1; if (!isa(BufferLocation->getOperand(ArgNo)) && !isa(BufferLocation->getOperand(ArgNo))) LocID = getMDOperandAsInt(BufferLocation, ArgNo); if (LocID >= 0) BA->addDecorate(DecorationBufferLocationINTEL, LocID); } if (RuntimeAligned && I->getType()->isPointerTy()) { // Order of integer numbers in MD node follows the order of function // parameters on which we shall attach the appropriate decoration. Add // decoration only if MD value is 1. int IsRuntimeAligned = 0; if (!isa(RuntimeAligned->getOperand(ArgNo)) && !isa(RuntimeAligned->getOperand(ArgNo))) IsRuntimeAligned = getMDOperandAsInt(RuntimeAligned, ArgNo); if (IsRuntimeAligned == 1) { // TODO: to replace non-conformant to the spec decoration generation // with: // BM->addExtension(ExtensionID::SPV_INTEL_runtime_aligned); // BM->addCapability(CapabilityRuntimeAlignedAttributeINTEL); // BA->addAttr(FunctionParameterAttributeRuntimeAlignedINTEL); BA->addDecorate(internal::DecorationRuntimeAlignedINTEL, IsRuntimeAligned); } } } if (Attrs.hasRetAttr(Attribute::ZExt)) BF->addDecorate(DecorationFuncParamAttr, FunctionParameterAttributeZext); if (Attrs.hasRetAttr(Attribute::SExt)) BF->addDecorate(DecorationFuncParamAttr, FunctionParameterAttributeSext); if (Attrs.hasFnAttr("referenced-indirectly")) { assert(!isKernel(F) && "kernel function was marked as referenced-indirectly"); BF->addDecorate(DecorationReferencedIndirectlyINTEL); } if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) transVectorComputeMetadata(F); transFPGAFunctionMetadata(BF, F); if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_maximum_registers)) transFunctionMetadataAsExecutionMode(BF, F); transAuxDataInst(BF, F); SPIRVDBG(dbgs() << "[transFunction] " << *F << " => "; spvdbgs() << *BF << '\n';) return BF; } void LLVMToSPIRVBase::transVectorComputeMetadata(Function *F) { using namespace VectorComputeUtil; if (!BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) return; auto BF = static_cast(getTranslatedValue(F)); assert(BF && "The SPIRVFunction pointer shouldn't be nullptr"); auto Attrs = F->getAttributes(); if (Attrs.hasFnAttr(kVCMetadata::VCStackCall)) BF->addDecorate(DecorationStackCallINTEL); if (Attrs.hasFnAttr(kVCMetadata::VCFunction)) BF->addDecorate(DecorationVectorComputeFunctionINTEL); if (Attrs.hasFnAttr(kVCMetadata::VCSIMTCall)) { SPIRVWord SIMTMode = 0; Attrs.getFnAttr(kVCMetadata::VCSIMTCall) .getValueAsString() .getAsInteger(0, SIMTMode); BF->addDecorate(DecorationSIMTCallINTEL, SIMTMode); } if (Attrs.hasRetAttr(kVCMetadata::VCSingleElementVector)) translateSEVDecoration( Attrs.getAttributeAtIndex(AttributeList::ReturnIndex, kVCMetadata::VCSingleElementVector), BF); for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { auto ArgNo = I->getArgNo(); SPIRVFunctionParameter *BA = BF->getArgument(ArgNo); if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCArgumentIOKind)) { SPIRVWord Kind = {}; Attrs.getParamAttr(ArgNo, kVCMetadata::VCArgumentIOKind) .getValueAsString() .getAsInteger(0, Kind); BA->addDecorate(DecorationFuncParamIOKindINTEL, Kind); } if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCSingleElementVector)) translateSEVDecoration( Attrs.getParamAttr(ArgNo, kVCMetadata::VCSingleElementVector), BA); if (Attrs.hasParamAttr(ArgNo, kVCMetadata::VCMediaBlockIO)) { assert(BA->getType()->isTypeImage() && "VCMediaBlockIO attribute valid only on image parameters"); BA->addDecorate(DecorationMediaBlockIOINTEL); } } if (!isKernel(F) && BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_float_controls2) && Attrs.hasFnAttr(kVCMetadata::VCFloatControl)) { SPIRVWord Mode = 0; Attrs.getFnAttr(kVCMetadata::VCFloatControl) .getValueAsString() .getAsInteger(0, Mode); VCFloatTypeSizeMap::foreach ( [&](VCFloatType FloatType, unsigned TargetWidth) { BF->addDecorate(new SPIRVDecorateFunctionDenormModeINTEL( BF, TargetWidth, getFPDenormMode(Mode, FloatType))); BF->addDecorate(new SPIRVDecorateFunctionRoundingModeINTEL( BF, TargetWidth, getFPRoundingMode(Mode))); BF->addDecorate(new SPIRVDecorateFunctionFloatingPointModeINTEL( BF, TargetWidth, getFPOperationMode(Mode))); }); } } void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F) { if (MDNode *StallEnable = F->getMetadata(kSPIR2MD::StallEnable)) { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_cluster_attributes)) { if (getMDOperandAsInt(StallEnable, 0)) BF->addDecorate(new SPIRVDecorateStallEnableINTEL(BF)); } } if (MDNode *LoopFuse = F->getMetadata(kSPIR2MD::LoopFuse)) { if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_loop_fuse)) { size_t Depth = getMDOperandAsInt(LoopFuse, 0); size_t Independent = getMDOperandAsInt(LoopFuse, 1); BF->addDecorate( new SPIRVDecorateFuseLoopsInFunctionINTEL(BF, Depth, Independent)); } } if (MDNode *PreferDSP = F->getMetadata(kSPIR2MD::PreferDSP)) { if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_dsp_control)) { size_t Mode = getMDOperandAsInt(PreferDSP, 0); MDNode *PropDSPPref = F->getMetadata(kSPIR2MD::PropDSPPref); size_t Propagate = PropDSPPref ? getMDOperandAsInt(PropDSPPref, 0) : 0; BF->addDecorate(new SPIRVDecorateMathOpDSPModeINTEL(BF, Mode, Propagate)); } } if (MDNode *InitiationInterval = F->getMetadata(kSPIR2MD::InitiationInterval)) { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) { if (size_t Cycles = getMDOperandAsInt(InitiationInterval, 0)) BF->addDecorate(new SPIRVDecorateInitiationIntervalINTEL(BF, Cycles)); } } if (MDNode *MaxConcurrency = F->getMetadata(kSPIR2MD::MaxConcurrency)) { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) { size_t Invocations = getMDOperandAsInt(MaxConcurrency, 0); BF->addDecorate(new SPIRVDecorateMaxConcurrencyINTEL(BF, Invocations)); } } if (MDNode *DisableLoopPipelining = F->getMetadata(kSPIR2MD::DisableLoopPipelining)) { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes)) { size_t Disable = getMDOperandAsInt(DisableLoopPipelining, 0); BF->addDecorate(new SPIRVDecoratePipelineEnableINTEL(BF, !Disable)); } } } void LLVMToSPIRVBase::transFunctionMetadataAsExecutionMode(SPIRVFunction *BF, Function *F) { SmallVector RegisterAllocModeMDs; F->getMetadata("RegisterAllocMode", RegisterAllocModeMDs); for (unsigned I = 0; I < RegisterAllocModeMDs.size(); I++) { auto *RegisterAllocMode = RegisterAllocModeMDs[I]->getOperand(0).get(); if (isa(RegisterAllocMode)) { const StringRef Str = getMDOperandAsString(RegisterAllocModeMDs[I], 0); const NamedMaximumNumberOfRegisters NamedValue = SPIRVNamedMaximumNumberOfRegistersNameMap::rmap(Str.str()); BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, ExecutionModeNamedMaximumRegistersINTEL, NamedValue))); } else if (isa(RegisterAllocMode)) { auto *RegisterAllocNodeMDOp = getMDOperandAsMDNode(RegisterAllocModeMDs[I], 0); const int Num = getMDOperandAsInt(RegisterAllocNodeMDOp, 0); auto *Const = BM->addConstant(transType(Type::getInt32Ty(F->getContext())), Num); BF->addExecutionMode(BM->add(new SPIRVExecutionModeId( BF, ExecutionModeMaximumRegistersIdINTEL, Const->getId()))); } else { const int64_t RegisterAllocVal = mdconst::dyn_extract(RegisterAllocMode)->getZExtValue(); BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, ExecutionModeMaximumRegistersINTEL, RegisterAllocVal))); } } } void LLVMToSPIRVBase::transAuxDataInst(SPIRVFunction *BF, Function *F) { auto *BM = BF->getModule(); if (!BM->preserveAuxData()) return; if (!BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) BM->addExtension(SPIRV::ExtensionID::SPV_KHR_non_semantic_info); else BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_6); const auto &FnAttrs = F->getAttributes().getFnAttrs(); for (const auto &Attr : FnAttrs) { std::vector Ops; Ops.push_back(BF->getId()); if (Attr.isStringAttribute()) { // Format for String attributes is: // NonSemanticAuxDataFunctionAttribute Fcn AttrName AttrValue // or, if no value: // NonSemanticAuxDataFunctionAttribute Fcn AttrName // // AttrName and AttrValue are always Strings const StringRef AttrKind = Attr.getKindAsString(); const StringRef AttrValue = Attr.getValueAsString(); auto *KindSpvString = BM->getString(AttrKind.str()); Ops.push_back(KindSpvString->getId()); if (!AttrValue.empty()) { auto *ValueSpvString = BM->getString(AttrValue.str()); Ops.push_back(ValueSpvString->getId()); } } else { // Format for other types is: // NonSemanticAuxDataFunctionAttribute Fcn AttrStr // AttrStr is always a String. const std::string AttrStr = Attr.getAsString(); auto *AttrSpvString = BM->getString(AttrStr); Ops.push_back(AttrSpvString->getId()); } BM->addAuxData(NonSemanticAuxData::FunctionAttribute, transType(Type::getVoidTy(F->getContext())), Ops); } SmallVector> AllMD; SmallVector MDNames; F->getContext().getMDKindNames(MDNames); F->getAllMetadata(AllMD); for (auto MD : AllMD) { const std::string MDName = MDNames[MD.first].str(); // spirv.Decorations, spirv.ParameterDecorations and debug information are // handled elsewhere for both forward and reverse translation and are // complicated to support here, so just skip them. if (MDName == SPIRV_MD_DECORATIONS || MDName == SPIRV_MD_PARAMETER_DECORATIONS || MD.first == LLVMContext::MD_dbg) continue; // Format for metadata is: // NonSemanticAuxDataFunctionMetadata Fcn MDName MDVals... // MDName is always a String, MDVals have different types as explained // below. Also note this instruction has a variable number of operands std::vector Ops; Ops.push_back(BF->getId()); Ops.push_back(BM->getString(MDName)->getId()); for (unsigned int OpIdx = 0; OpIdx < MD.second->getNumOperands(); OpIdx++) { const auto &CurOp = MD.second->getOperand(OpIdx); if (auto *MDStr = dyn_cast(CurOp)) { // For MDString, MDVal is String auto *SPIRVStr = BM->getString(MDStr->getString().str()); Ops.push_back(SPIRVStr->getId()); } else if (auto *ValueAsMeta = dyn_cast(CurOp)) { // For Value metadata, MDVal is a SPIRVValue auto *SPIRVVal = transValue(ValueAsMeta->getValue(), nullptr); Ops.push_back(SPIRVVal->getId()); } else { assert(false && "Unsupported metadata type"); } } BM->addAuxData(NonSemanticAuxData::FunctionMetadata, transType(Type::getVoidTy(F->getContext())), Ops); } } SPIRVValue *LLVMToSPIRVBase::transConstant(Value *V) { if (auto CPNull = dyn_cast(V)) return BM->addNullConstant( bcast(transType(CPNull->getType()))); if (auto CAZero = dyn_cast(V)) { Type *AggType = CAZero->getType(); if (const StructType *ST = dyn_cast(AggType)) if (ST->hasName() && ST->getName() == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler)) return BM->addSamplerConstant(transType(AggType), 0, 0, 0); return BM->addNullConstant(transType(AggType)); } if (auto ConstI = dyn_cast(V)) { unsigned BitWidth = ConstI->getType()->getBitWidth(); if (BitWidth > 64) { BM->getErrorLog().checkError( BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_integers), SPIRVEC_InvalidBitWidth, std::to_string(BitWidth)); return BM->addConstant(transType(V->getType()), ConstI->getValue()); } return BM->addConstant(transType(V->getType()), ConstI->getZExtValue()); } if (auto ConstFP = dyn_cast(V)) { auto BT = static_cast(transType(V->getType())); return BM->addConstant( BT, ConstFP->getValueAPF().bitcastToAPInt().getZExtValue()); } if (auto ConstDA = dyn_cast(V)) { std::vector BV; for (unsigned I = 0, E = ConstDA->getNumElements(); I != E; ++I) BV.push_back(transValue(ConstDA->getElementAsConstant(I), nullptr, true, FuncTransMode::Pointer)); return BM->addCompositeConstant(transType(V->getType()), BV); } if (auto ConstA = dyn_cast(V)) { std::vector BV; for (auto I = ConstA->op_begin(), E = ConstA->op_end(); I != E; ++I) BV.push_back(transValue(*I, nullptr, true, FuncTransMode::Pointer)); return BM->addCompositeConstant(transType(V->getType()), BV); } if (auto ConstDV = dyn_cast(V)) { std::vector BV; for (unsigned I = 0, E = ConstDV->getNumElements(); I != E; ++I) BV.push_back(transValue(ConstDV->getElementAsConstant(I), nullptr, true, FuncTransMode::Pointer)); return BM->addCompositeConstant(transType(V->getType()), BV); } if (auto ConstV = dyn_cast(V)) { std::vector BV; for (auto I = ConstV->op_begin(), E = ConstV->op_end(); I != E; ++I) BV.push_back(transValue(*I, nullptr, true, FuncTransMode::Pointer)); return BM->addCompositeConstant(transType(V->getType()), BV); } if (const auto *ConstV = dyn_cast(V)) { StringRef StructName; if (ConstV->getType()->hasName()) StructName = ConstV->getType()->getName(); if (StructName == getSPIRVTypeName(kSPIRVTypeName::ConstantSampler)) { assert(ConstV->getNumOperands() == 3); SPIRVWord AddrMode = ConstV->getOperand(0)->getUniqueInteger().getZExtValue(), Normalized = ConstV->getOperand(1)->getUniqueInteger().getZExtValue(), FilterMode = ConstV->getOperand(2)->getUniqueInteger().getZExtValue(); assert(AddrMode < 5 && "Invalid addressing mode"); assert(Normalized < 2 && "Invalid value of normalized coords"); assert(FilterMode < 2 && "Invalid filter mode"); SPIRVType *SamplerTy = transType(ConstV->getType()); return BM->addSamplerConstant(SamplerTy, AddrMode, Normalized, FilterMode); } if (StructName == getSPIRVTypeName(kSPIRVTypeName::ConstantPipeStorage)) { assert(ConstV->getNumOperands() == 3); SPIRVWord PacketSize = ConstV->getOperand(0)->getUniqueInteger().getZExtValue(), PacketAlign = ConstV->getOperand(1)->getUniqueInteger().getZExtValue(), Capacity = ConstV->getOperand(2)->getUniqueInteger().getZExtValue(); assert(PacketAlign >= 1 && "Invalid packet alignment"); assert(PacketSize >= PacketAlign && PacketSize % PacketAlign == 0 && "Invalid packet size and/or alignment."); SPIRVType *PipeStorageTy = transType(ConstV->getType()); return BM->addPipeStorageConstant(PipeStorageTy, PacketSize, PacketAlign, Capacity); } std::vector BV; for (auto I = ConstV->op_begin(), E = ConstV->op_end(); I != E; ++I) BV.push_back(transValue(*I, nullptr, true, FuncTransMode::Pointer)); return BM->addCompositeConstant(transType(V->getType()), BV); } if (auto ConstUE = dyn_cast(V)) { auto Inst = ConstUE->getAsInstruction(); SPIRVDBG(dbgs() << "ConstantExpr: " << *ConstUE << '\n'; dbgs() << "Instruction: " << *Inst << '\n';) auto BI = transValue(Inst, nullptr, false); Inst->dropAllReferences(); UnboundInst.push_back(Inst); return BI; } if (isa(V)) { return BM->addUndef(transType(V->getType())); } return nullptr; } SPIRVValue *LLVMToSPIRVBase::transValue(Value *V, SPIRVBasicBlock *BB, bool CreateForward, FuncTransMode FuncTrans) { LLVMToSPIRVValueMap::iterator Loc = ValueMap.find(V); if (Loc != ValueMap.end() && (!Loc->second->isForward() || CreateForward) && // do not return forward-decl of a function if we // actually want to create a function pointer !(FuncTrans == FuncTransMode::Pointer && isa(V))) return Loc->second; SPIRVDBG(dbgs() << "[transValue] " << *V << '\n'); assert((!isa(V) || isa(V) || isa(V) || isa(V) || isa(V) || BB) && "Invalid SPIRV BB"); auto *BV = transValueWithoutDecoration(V, BB, CreateForward, FuncTrans); if (!BV) return nullptr; // Only translate decorations for non-forward instructions. Forward // instructions will have their decorations translated when the actual // instruction is seen and rewritten to a real SPIR-V instruction. if (!BV->isForward() && !transDecoration(V, BV)) return nullptr; StringRef Name = V->getName(); if (!Name.empty()) // Don't erase the name, which BM might already have BM->setName(BV, Name.str()); return BV; } SPIRVInstruction *LLVMToSPIRVBase::transBinaryInst(BinaryOperator *B, SPIRVBasicBlock *BB) { unsigned LLVMOC = B->getOpcode(); auto Op0 = transValue(B->getOperand(0), BB); SPIRVInstruction *BI = BM->addBinaryInst( transBoolOpCode(Op0, OpCodeMap::map(LLVMOC)), transType(B->getType()), Op0, transValue(B->getOperand(1), BB), BB); // BinaryOperator can have no parent if it is handled as an expression inside // another instruction. if (B->getParent() && isUnfusedMulAdd(B)) { Function *F = B->getFunction(); SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName() << ": possible fma candidate " << *B << '\n'); joinFPContract(F, FPContract::DISABLED); } return BI; } SPIRVInstruction *LLVMToSPIRVBase::transCmpInst(CmpInst *Cmp, SPIRVBasicBlock *BB) { auto *Op0 = Cmp->getOperand(0); SPIRVValue *TOp0 = transValue(Op0, BB); SPIRVValue *TOp1 = transValue(Cmp->getOperand(1), BB); if (Op0->getType()->isPointerTy()) { auto P = Cmp->getPredicate(); if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4) && (P == ICmpInst::ICMP_EQ || P == ICmpInst::ICMP_NE) && Cmp->getOperand(1)->getType()->isPointerTy()) { const Op OC = P == ICmpInst::ICMP_EQ ? OpPtrEqual : OpPtrNotEqual; return BM->addBinaryInst(OC, transType(Cmp->getType()), TOp0, TOp1, BB); } unsigned AS = cast(Op0->getType())->getAddressSpace(); SPIRVType *Ty = transType(getSizetType(AS)); TOp0 = BM->addUnaryInst(OpConvertPtrToU, Ty, TOp0, BB); TOp1 = BM->addUnaryInst(OpConvertPtrToU, Ty, TOp1, BB); } SPIRVInstruction *BI = BM->addCmpInst(transBoolOpCode(TOp0, CmpMap::map(Cmp->getPredicate())), transType(Cmp->getType()), TOp0, TOp1, BB); return BI; } SPIRVValue *LLVMToSPIRVBase::transUnaryInst(UnaryInstruction *U, SPIRVBasicBlock *BB) { if (isa(U) && U->getType()->isPointerTy()) { if (isa(U->getOperand(0))) { SPIRVType *ExpectedTy = transScavengedType(U); return BM->addNullConstant(bcast(ExpectedTy)); } if (isa(U->getOperand(0))) { SPIRVType *ExpectedTy = transScavengedType(U); return BM->addUndef(ExpectedTy); } } Op BOC = OpNop; if (auto Cast = dyn_cast(U)) { const auto SrcAddrSpace = Cast->getSrcTy()->getPointerAddressSpace(); const auto DestAddrSpace = Cast->getDestTy()->getPointerAddressSpace(); if (DestAddrSpace == SPIRAS_Generic) { getErrorLog().checkError( SrcAddrSpace != SPIRAS_Constant, SPIRVEC_InvalidModule, U, "Casts from constant address space to generic are illegal\n"); BOC = OpPtrCastToGeneric; // In SPIR-V only casts to/from generic are allowed. But with // SPV_INTEL_usm_storage_classes we can also have casts from global_device // and global_host to global addr space and vice versa. } else if (SrcAddrSpace == SPIRAS_GlobalDevice || SrcAddrSpace == SPIRAS_GlobalHost) { getErrorLog().checkError(DestAddrSpace == SPIRAS_Global || DestAddrSpace == SPIRAS_Generic, SPIRVEC_InvalidModule, U, "Casts from global_device/global_host only " "allowed to global/generic\n"); if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_usm_storage_classes)) { if (DestAddrSpace == SPIRAS_Global) return nullptr; BOC = OpPtrCastToGeneric; } else { BOC = OpPtrCastToCrossWorkgroupINTEL; } } else if (DestAddrSpace == SPIRAS_GlobalDevice || DestAddrSpace == SPIRAS_GlobalHost) { getErrorLog().checkError(SrcAddrSpace == SPIRAS_Global || SrcAddrSpace == SPIRAS_Generic, SPIRVEC_InvalidModule, U, "Casts to global_device/global_host only " "allowed from global/generic\n"); if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_usm_storage_classes)) { if (SrcAddrSpace == SPIRAS_Global) return nullptr; BOC = OpGenericCastToPtr; } else { BOC = OpCrossWorkgroupCastToPtrINTEL; } } else { getErrorLog().checkError( SrcAddrSpace == SPIRAS_Generic, SPIRVEC_InvalidModule, U, "Casts from private/local/global address space are allowed only to " "generic\n"); getErrorLog().checkError( DestAddrSpace != SPIRAS_Constant, SPIRVEC_InvalidModule, U, "Casts from generic address space to constant are illegal\n"); BOC = OpGenericCastToPtr; } } else { auto OpCode = U->getOpcode(); BOC = OpCodeMap::map(OpCode); } auto Op = transValue(U->getOperand(0), BB, true, FuncTransMode::Pointer); SPIRVType *TransTy = transScavengedType(U); return BM->addUnaryInst(transBoolOpCode(Op, BOC), TransTy, Op, BB); } /// This helper class encapsulates information extraction from /// "llvm.loop.parallel_access_indices" metadata hints. Initialize /// with a pointer to an MDNode with the following structure: /// ! = !{!"llvm.loop.parallel_access_indices", !, !, ...} /// OR: /// ! = !{!"llvm.loop.parallel_access_indices", !, i32 } /// /// All of the MDNode-type operands mark the index groups for particular /// array variables. An optional i32 value indicates the safelen (safe /// number of iterations) for the optimization application to these /// array variables. If the safelen value is absent, an infinite /// number of iterations is implied. class LLVMParallelAccessIndices { public: LLVMParallelAccessIndices( MDNode *Node, LLVMToSPIRVBase::LLVMToSPIRVMetadataMap &IndexGroupArrayMap) : Node(Node), IndexGroupArrayMap(IndexGroupArrayMap) {} void initialize() { assert(isValid() && "LLVMParallelAccessIndices initialized from an invalid MDNode"); unsigned NumOperands = Node->getNumOperands(); auto *SafeLenExpression = mdconst::dyn_extract_or_null( Node->getOperand(NumOperands - 1)); // If no safelen value is specified and the last operand // casts to an MDNode* rather than an int, 0 will be stored SafeLen = SafeLenExpression ? SafeLenExpression->getZExtValue() : 0; // Count MDNode operands that refer to index groups: // - operand [0] is a string literal and should be ignored; // - depending on whether a particular safelen is specified as the // last operand, we may or may not want to extract the latter // as an index group unsigned NumIdxGroups = SafeLen ? NumOperands - 2 : NumOperands - 1; for (unsigned I = 1; I <= NumIdxGroups; ++I) { MDNode *IdxGroupNode = getMDOperandAsMDNode(Node, I); assert(IdxGroupNode && "Invalid operand in the MDNode for LLVMParallelAccessIndices"); auto IdxGroupArrayPairIt = IndexGroupArrayMap.find(IdxGroupNode); // TODO: Some LLVM IR optimizations (e.g. loop inlining as part of // the function inlining) can result in invalid parallel_access_indices // metadata. Only valid cases will pass the subsequent check and // survive the translation. This check should be replaced with an // assertion once all known cases are handled. if (IdxGroupArrayPairIt != IndexGroupArrayMap.end()) for (SPIRVId ArrayAccessId : IdxGroupArrayPairIt->second) ArrayVariablesVec.push_back(ArrayAccessId); } } bool isValid() { bool IsNamedCorrectly = getMDOperandAsString(Node, 0) == ExpectedName; return Node && IsNamedCorrectly; } unsigned getSafeLen() { return SafeLen; } const std::vector &getArrayVariables() { return ArrayVariablesVec; } private: MDNode *Node; LLVMToSPIRVBase::LLVMToSPIRVMetadataMap &IndexGroupArrayMap; const std::string ExpectedName = "llvm.loop.parallel_access_indices"; std::vector ArrayVariablesVec; unsigned SafeLen; }; /// Go through the operands !llvm.loop metadata attached to the branch /// instruction, fill the Loop Control mask and possible parameters for its /// fields. spv::LoopControlMask LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch, std::vector &Parameters) { if (!Branch) return spv::LoopControlMaskNone; MDNode *LoopMD = Branch->getMetadata("llvm.loop"); if (!LoopMD) return spv::LoopControlMaskNone; size_t LoopControl = spv::LoopControlMaskNone; std::vector> ParametersToSort; // If only a subset of loop count parameters is defined in metadata // then undefined ones should have a default value -1 in SPIR-V. // Preset all loop count parameters with the default value. struct LoopCountInfo { int64_t Min = -1, Max = -1, Avg = -1; } LoopCount; // Unlike with most of the cases, some loop metadata specifications // can occur multiple times - for these, all correspondent tokens // need to be collected first, and only then added to SPIR-V loop // parameters in a separate routine std::vector> DependencyArrayParameters; for (const MDOperand &MDOp : LoopMD->operands()) { if (MDNode *Node = dyn_cast(MDOp)) { StringRef S = getMDOperandAsString(Node, 0); // Set the loop control bits. Parameters are set in the order described // in 3.23 SPIR-V Spec. rev. 1.4: // Bits that are set can indicate whether an additional operand follows, // as described by the table. If there are multiple following operands // indicated, they are ordered: Those indicated by smaller-numbered bits // appear first. if (S == "llvm.loop.unroll.disable") LoopControl |= spv::LoopControlDontUnrollMask; else if (S == "llvm.loop.unroll.enable") LoopControl |= spv::LoopControlUnrollMask; // Attempt to do full unroll of the loop and disable unrolling if the trip // count is not known at compile time by setting PartialCount to 1 else if (S == "llvm.loop.unroll.full") { LoopControl |= spv::LoopControlUnrollMask; if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) { BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); ParametersToSort.emplace_back(spv::LoopControlPartialCountMask, 1); LoopControl |= spv::LoopControlPartialCountMask; } } // PartialCount must not be used with the DontUnroll bit else if (S == "llvm.loop.unroll.count" && !(LoopControl & LoopControlDontUnrollMask)) { if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) { BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back(spv::LoopControlPartialCountMask, I); LoopControl |= spv::LoopControlPartialCountMask; } } else if (S == "llvm.loop.ivdep.enable") LoopControl |= spv::LoopControlDependencyInfiniteMask; else if (S == "llvm.loop.ivdep.safelen") { size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back(spv::LoopControlDependencyLengthMask, I); LoopControl |= spv::LoopControlDependencyLengthMask; } else if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_loop_controls)) { // Add Intel specific Loop Control masks if (S == "llvm.loop.ii.count") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back( spv::LoopControlInitiationIntervalINTELMask, I); LoopControl |= spv::LoopControlInitiationIntervalINTELMask; } else if (S == "llvm.loop.max_concurrency.count") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back(spv::LoopControlMaxConcurrencyINTELMask, I); LoopControl |= spv::LoopControlMaxConcurrencyINTELMask; } else if (S == "llvm.loop.parallel_access_indices") { // Intel FPGA IVDep loop attribute LLVMParallelAccessIndices IVDep(Node, IndexGroupArrayMap); IVDep.initialize(); // Store IVDep-specific parameters into an intermediate // container to address the case when there're multiple // IVDep metadata nodes and this condition gets entered multiple // times. The update of the main parameters vector & the loop control // mask will be done later, in the main scope of the function unsigned SafeLen = IVDep.getSafeLen(); for (auto &ArrayId : IVDep.getArrayVariables()) DependencyArrayParameters.emplace_back(ArrayId, SafeLen); } else if (S == "llvm.loop.intel.pipelining.enable") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back(spv::LoopControlPipelineEnableINTELMask, I); LoopControl |= spv::LoopControlPipelineEnableINTELMask; } else if (S == "llvm.loop.coalesce.enable") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); ParametersToSort.emplace_back(spv::LoopControlLoopCoalesceINTELMask, 0); LoopControl |= spv::LoopControlLoopCoalesceINTELMask; } else if (S == "llvm.loop.coalesce.count") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back(spv::LoopControlLoopCoalesceINTELMask, I); LoopControl |= spv::LoopControlLoopCoalesceINTELMask; } else if (S == "llvm.loop.max_interleaving.count") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back( spv::LoopControlMaxInterleavingINTELMask, I); LoopControl |= spv::LoopControlMaxInterleavingINTELMask; } else if (S == "llvm.loop.intel.speculated.iterations.count") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); size_t I = getMDOperandAsInt(Node, 1); ParametersToSort.emplace_back( spv::LoopControlSpeculatedIterationsINTELMask, I); LoopControl |= spv::LoopControlSpeculatedIterationsINTELMask; } else if (S == "llvm.loop.fusion.disable") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); LoopControl |= spv::LoopControlNoFusionINTELMask; } else if (S == "llvm.loop.intel.loopcount_min") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); LoopCount.Min = getMDOperandAsInt(Node, 1); LoopControl |= spv::internal::LoopControlLoopCountINTELMask; } else if (S == "llvm.loop.intel.loopcount_max") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); LoopCount.Max = getMDOperandAsInt(Node, 1); LoopControl |= spv::internal::LoopControlLoopCountINTELMask; } else if (S == "llvm.loop.intel.loopcount_avg") { BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); LoopCount.Avg = getMDOperandAsInt(Node, 1); LoopControl |= spv::internal::LoopControlLoopCountINTELMask; } } } } if (LoopControl & spv::internal::LoopControlLoopCountINTELMask) { // LoopCountINTELMask have int64 literal parameters and we need to store // int64 into 2 SPIRVWords ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Min)); ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Min >> 32)); ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Max)); ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Max >> 32)); ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Avg)); ParametersToSort.emplace_back(spv::internal::LoopControlLoopCountINTELMask, static_cast(LoopCount.Avg >> 32)); } // If any loop control parameters were held back until fully collected, // now is the time to move the information to the main parameters collection if (!DependencyArrayParameters.empty()) { // The first parameter states the number of pairs to be // listed ParametersToSort.emplace_back(spv::LoopControlDependencyArrayINTELMask, DependencyArrayParameters.size()); for (auto &ArraySflnPair : DependencyArrayParameters) { ParametersToSort.emplace_back(spv::LoopControlDependencyArrayINTELMask, ArraySflnPair.first); ParametersToSort.emplace_back(spv::LoopControlDependencyArrayINTELMask, ArraySflnPair.second); } BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls); BM->addCapability(CapabilityFPGALoopControlsINTEL); LoopControl |= spv::LoopControlDependencyArrayINTELMask; } std::stable_sort(ParametersToSort.begin(), ParametersToSort.end(), [](const std::pair &CompareLeft, const std::pair &CompareRight) { return CompareLeft.first < CompareRight.first; }); for (auto Param : ParametersToSort) Parameters.push_back(Param.second); return static_cast(LoopControl); } static int transAtomicOrdering(llvm::AtomicOrdering Ordering) { return OCLMemOrderMap::map( static_cast(llvm::toCABI(Ordering))); } SPIRVValue *LLVMToSPIRVBase::transAtomicStore(StoreInst *ST, SPIRVBasicBlock *BB) { std::vector Ops{ST->getPointerOperand(), getUInt32(M, spv::ScopeDevice), getUInt32(M, transAtomicOrdering(ST->getOrdering())), ST->getValueOperand()}; std::vector SPIRVOps = transValue(Ops, BB); return mapValue(ST, BM->addInstTemplate(OpAtomicStore, BM->getIds(SPIRVOps), BB, nullptr)); } SPIRVValue *LLVMToSPIRVBase::transAtomicLoad(LoadInst *LD, SPIRVBasicBlock *BB) { std::vector Ops{ LD->getPointerOperand(), getUInt32(M, spv::ScopeDevice), getUInt32(M, transAtomicOrdering(LD->getOrdering()))}; std::vector SPIRVOps = transValue(Ops, BB); return mapValue(LD, BM->addInstTemplate(OpAtomicLoad, BM->getIds(SPIRVOps), BB, transType(LD->getType()))); } // Aliasing list MD contains several scope MD nodes whithin it. Each scope MD // has a selfreference and an extra MD node for aliasing domain and also it // can contain an optional string operand. Domain MD contains a self-reference // with an optional string operand. Here we unfold the list, creating SPIR-V // aliasing instructions. // TODO: add support for an optional string operand. SPIRVEntry *addMemAliasingINTELInstructions(SPIRVModule *M, MDNode *AliasingListMD) { if (AliasingListMD->getNumOperands() == 0) return nullptr; std::vector ListId; for (const MDOperand &MDListOp : AliasingListMD->operands()) { if (MDNode *ScopeMD = dyn_cast(MDListOp)) { if (ScopeMD->getNumOperands() < 2) return nullptr; MDNode *DomainMD = dyn_cast(ScopeMD->getOperand(1)); if (!DomainMD) return nullptr; auto *Domain = M->getOrAddAliasDomainDeclINTELInst(std::vector(), DomainMD); auto *Scope = M->getOrAddAliasScopeDeclINTELInst({Domain->getId()}, ScopeMD); ListId.push_back(Scope->getId()); } } return M->getOrAddAliasScopeListDeclINTELInst(ListId, AliasingListMD); } // Translate alias.scope/noalias metadata attached to store and load // instructions. void transAliasingMemAccess(SPIRVModule *BM, MDNode *AliasingListMD, std::vector &MemoryAccess, SPIRVWord MemAccessMask) { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_memory_access_aliasing)) return; auto *MemAliasList = addMemAliasingINTELInstructions(BM, AliasingListMD); if (!MemAliasList) return; MemoryAccess[0] |= MemAccessMask; MemoryAccess.push_back(MemAliasList->getId()); } /// An instruction may use an instruction from another BB which has not been /// translated. SPIRVForward should be created as place holder for these /// instructions and replaced later by the real instructions. /// Use CreateForward = true to indicate such situation. SPIRVValue * LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, bool CreateForward, FuncTransMode FuncTrans) { if (auto LBB = dyn_cast(V)) { auto BF = static_cast(getTranslatedValue(LBB->getParent())); assert(BF && "Function not translated"); BB = static_cast(mapValue(V, BM->addBasicBlock(BF))); BM->setName(BB, LBB->getName().str()); return BB; } if (auto *F = dyn_cast(V)) { if (FuncTrans == FuncTransMode::Decl) return transFunctionDecl(F); if (!BM->checkExtension(ExtensionID::SPV_INTEL_function_pointers, SPIRVEC_FunctionPointers, toString(V))) return nullptr; return BM->addConstantFunctionPointerINTEL( transPointerType(F->getFunctionType(), F->getAddressSpace()), static_cast(transValue(F, nullptr))); } if (auto GV = dyn_cast(V)) { llvm::Type *Ty = GV->getValueType(); // Though variables with common linkage type are initialized by 0, // they can be represented in SPIR-V as uninitialized variables with // 'Export' linkage type, just as tentative definitions look in C llvm::Value *Init = GV->hasInitializer() && !GV->hasCommonLinkage() ? GV->getInitializer() : nullptr; SPIRVValue *BVarInit = nullptr; StructType *ST = Init ? dyn_cast(Init->getType()) : nullptr; if (ST && ST->hasName() && isSPIRVConstantName(ST->getName())) { auto BV = transConstant(Init); assert(BV); return mapValue(V, BV); } else if (ConstantExpr *ConstUE = dyn_cast_or_null(Init)) { Value *SpecialInit = unwrapSpecialTypeInitializer(ConstUE); if (auto *SpecialGV = dyn_cast_or_null(SpecialInit)) { Init = SpecialGV; Ty = SpecialGV->getValueType(); } BVarInit = transValue(Init, nullptr); } else if (ST && isa(Init)) { // Undef initializer for LLVM structure be can translated to // OpConstantComposite with OpUndef constituents. auto I = ValueMap.find(Init); if (I == ValueMap.end()) { std::vector Elements; for (Type *E : ST->elements()) Elements.push_back(transValue(UndefValue::get(E), nullptr)); BVarInit = BM->addCompositeConstant(transType(ST), Elements); ValueMap[Init] = BVarInit; } else BVarInit = I->second; } else if (Init && !isa(Init)) { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_long_constant_composite)) { if (auto ArrTy = dyn_cast_or_null(Init->getType())) { // First 3 words of OpConstantComposite encode: 1) word count & // opcode, 2) Result Type and 3) Result Id. Max length of SPIRV // instruction = 65535 words. constexpr int MaxNumElements = MaxWordCount - SPIRVSpecConstantComposite::FixedWC; if (ArrTy->getNumElements() > MaxNumElements && !isa(Init)) { std::stringstream SS; SS << "Global variable has a constant array initializer with a " << "number of elements greater than OpConstantComposite can " << "have (" << MaxNumElements << "). Should the array be " << "split?\n Original LLVM value:\n" << toString(GV); getErrorLog().checkError(false, SPIRVEC_InvalidWordCount, SS.str()); } } } BVarInit = transValue(Init, nullptr); } SPIRVStorageClassKind StorageClass; auto AddressSpace = static_cast(GV->getAddressSpace()); bool IsVectorCompute = BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute) && GV->hasAttribute(kVCMetadata::VCGlobalVariable); if (IsVectorCompute) StorageClass = VectorComputeUtil::getVCGlobalVarStorageClass(AddressSpace); else { // Lower global_device and global_host address spaces that were added in // SYCL as part of SYCL_INTEL_usm_address_spaces extension to just global // address space if device doesn't support SPV_INTEL_usm_storage_classes // extension if ((AddressSpace == SPIRAS_GlobalDevice || AddressSpace == SPIRAS_GlobalHost) && !BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_usm_storage_classes)) AddressSpace = SPIRAS_Global; StorageClass = SPIRSPIRVAddrSpaceMap::map(AddressSpace); } SPIRVType *TranslatedTy = transPointerType(Ty, GV->getAddressSpace()); auto BVar = static_cast( BM->addVariable(TranslatedTy, GV->isConstant(), transLinkageType(GV), BVarInit, GV->getName().str(), StorageClass, nullptr)); if (IsVectorCompute) { BVar->addDecorate(DecorationVectorComputeVariableINTEL); if (GV->hasAttribute(kVCMetadata::VCByteOffset)) { SPIRVWord Offset = {}; GV->getAttribute(kVCMetadata::VCByteOffset) .getValueAsString() .getAsInteger(0, Offset); BVar->addDecorate(DecorationGlobalVariableOffsetINTEL, Offset); } if (GV->hasAttribute(kVCMetadata::VCVolatile)) BVar->addDecorate(DecorationVolatile); if (GV->hasAttribute(kVCMetadata::VCSingleElementVector)) translateSEVDecoration( GV->getAttribute(kVCMetadata::VCSingleElementVector), BVar); } mapValue(V, BVar); spv::BuiltIn Builtin = spv::BuiltInPosition; if (!GV->hasName() || !getSPIRVBuiltin(GV->getName().str(), Builtin)) return BVar; if (static_cast(Builtin) >= internal::BuiltInSubDeviceIDINTEL && static_cast(Builtin) <= internal::BuiltInGlobalHWThreadIDINTEL) { const std::string ErrorStr = "SPV_INTEL_hw_thread_queries\n" "Please report to " "https://github.com/intel/llvm in case if you see " "this error.\nRef LLVM Value:"; getErrorLog().checkError(false, SPIRVEC_DeprecatedExtension, V, ErrorStr); } BVar->setBuiltin(Builtin); return BVar; } if (isa(V)) { auto BV = transConstant(V); assert(BV); // Don't store opaque pointer constants in the map--we might reuse the wrong // type for (e.g.) a null value if we do so. if (V->getType()->isOpaquePointerTy()) return BV; return mapValue(V, BV); } if (auto Arg = dyn_cast(V)) { unsigned ArgNo = Arg->getArgNo(); SPIRVFunction *BF = BB->getParent(); // assert(BF->existArgument(ArgNo)); return mapValue(V, BF->getArgument(ArgNo)); } if (CreateForward) return mapValue(V, BM->addForward(transScavengedType(V))); if (StoreInst *ST = dyn_cast(V)) { if (ST->isAtomic()) return transAtomicStore(ST, BB); // Keep this vector to store MemoryAccess operands for both Alignment and // Aliasing information. std::vector MemoryAccess(1, 0); if (ST->isVolatile()) MemoryAccess[0] |= MemoryAccessVolatileMask; MemoryAccess[0] |= MemoryAccessAlignedMask; MemoryAccess.push_back(ST->getAlign().value()); if (ST->getMetadata(LLVMContext::MD_nontemporal)) MemoryAccess[0] |= MemoryAccessNontemporalMask; if (MDNode *AliasingListMD = ST->getMetadata(LLVMContext::MD_alias_scope)) transAliasingMemAccess(BM, AliasingListMD, MemoryAccess, MemoryAccessAliasScopeINTELMaskMask); if (MDNode *AliasingListMD = ST->getMetadata(LLVMContext::MD_noalias)) transAliasingMemAccess(BM, AliasingListMD, MemoryAccess, MemoryAccessNoAliasINTELMaskMask); if (MemoryAccess.front() == 0) MemoryAccess.clear(); return mapValue(V, BM->addStoreInst(transValue(ST->getPointerOperand(), BB), transValue(ST->getValueOperand(), BB, true, FuncTransMode::Pointer), MemoryAccess, BB)); } if (LoadInst *LD = dyn_cast(V)) { if (LD->isAtomic()) return transAtomicLoad(LD, BB); // Keep this vector to store MemoryAccess operands for both Alignment and // Aliasing information. std::vector MemoryAccess(1, 0); if (LD->isVolatile()) MemoryAccess[0] |= MemoryAccessVolatileMask; MemoryAccess[0] |= MemoryAccessAlignedMask; MemoryAccess.push_back(LD->getAlign().value()); if (LD->getMetadata(LLVMContext::MD_nontemporal)) MemoryAccess[0] |= MemoryAccessNontemporalMask; if (MDNode *AliasingListMD = LD->getMetadata(LLVMContext::MD_alias_scope)) transAliasingMemAccess(BM, AliasingListMD, MemoryAccess, MemoryAccessAliasScopeINTELMaskMask); if (MDNode *AliasingListMD = LD->getMetadata(LLVMContext::MD_noalias)) transAliasingMemAccess(BM, AliasingListMD, MemoryAccess, MemoryAccessNoAliasINTELMaskMask); if (MemoryAccess.front() == 0) MemoryAccess.clear(); return mapValue(V, BM->addLoadInst(transValue(LD->getPointerOperand(), BB), MemoryAccess, BB)); } if (BinaryOperator *B = dyn_cast(V)) { SPIRVInstruction *BI = transBinaryInst(B, BB); return mapValue(V, BI); } if (dyn_cast(V)) return mapValue(V, BM->addUnreachableInst(BB)); if (auto RI = dyn_cast(V)) { if (auto RV = RI->getReturnValue()) return mapValue(V, BM->addReturnValueInst(transValue(RV, BB), BB)); return mapValue(V, BM->addReturnInst(BB)); } if (CmpInst *Cmp = dyn_cast(V)) { SPIRVInstruction *BI = transCmpInst(Cmp, BB); return mapValue(V, BI); } if (SelectInst *Sel = dyn_cast(V)) return mapValue( V, BM->addSelectInst( transValue(Sel->getCondition(), BB), transValue(Sel->getTrueValue(), BB, true, FuncTransMode::Pointer), transValue(Sel->getFalseValue(), BB, true, FuncTransMode::Pointer), BB)); if (AllocaInst *Alc = dyn_cast(V)) { SPIRVType *TranslatedTy = transPointerType(Alc->getAllocatedType(), Alc->getAddressSpace()); if (Alc->isArrayAllocation()) { if (!BM->checkExtension(ExtensionID::SPV_INTEL_variable_length_array, SPIRVEC_InvalidInstruction, toString(Alc) + "\nTranslation of dynamic alloca requires " "SPV_INTEL_variable_length_array extension.")) return nullptr; SPIRVValue *Length = transValue(Alc->getArraySize(), BB); assert(Length && "Couldn't translate array size!"); return mapValue(V, BM->addInstTemplate(OpVariableLengthArrayINTEL, {Length->getId()}, BB, TranslatedTy)); } return mapValue(V, BM->addVariable(TranslatedTy, false, spv::internal::LinkageTypeInternal, nullptr, Alc->getName().str(), StorageClassFunction, BB)); } if (auto *Switch = dyn_cast(V)) { std::vector Pairs; auto Select = transValue(Switch->getCondition(), BB); for (auto I = Switch->case_begin(), E = Switch->case_end(); I != E; ++I) { SPIRVSwitch::LiteralTy Lit; uint64_t CaseValue = I->getCaseValue()->getZExtValue(); Lit.push_back(CaseValue); assert(Select->getType()->getBitWidth() <= 64 && "unexpected selector bitwidth"); if (Select->getType()->getBitWidth() == 64) Lit.push_back(CaseValue >> 32); Pairs.push_back( std::make_pair(Lit, static_cast( transValue(I->getCaseSuccessor(), nullptr)))); } return mapValue( V, BM->addSwitchInst(Select, static_cast( transValue(Switch->getDefaultDest(), nullptr)), Pairs, BB)); } if (BranchInst *Branch = dyn_cast(V)) { SPIRVLabel *SuccessorTrue = static_cast(transValue(Branch->getSuccessor(0), BB)); /// Clang attaches !llvm.loop metadata to "latch" BB. This kind of blocks /// has an edge directed to the loop header. Thus latch BB matching to /// "Continue Target" per the SPIR-V spec. This statement is true only after /// applying the loop-simplify pass to the LLVM module. /// For "for" and "while" loops latch BB is terminated by an /// unconditional branch. Also for this kind of loops "Merge Block" can /// be found as block targeted by false edge of the "Header" BB. /// For "do while" loop the latch is terminated by a conditional branch /// with true edge going to the header and the false edge going out of /// the loop, which corresponds to a "Merge Block" per the SPIR-V spec. std::vector Parameters; spv::LoopControlMask LoopControl = getLoopControl(Branch, Parameters); if (Branch->isUnconditional()) { // Usually, "for" and "while" loops llvm.loop metadata is attached to an // unconditional branch instruction. if (LoopControl != spv::LoopControlMaskNone) { // SuccessorTrue is the loop header BB. const SPIRVInstruction *Term = SuccessorTrue->getTerminateInstr(); if (Term && Term->getOpCode() == OpBranchConditional) { const auto *Br = static_cast(Term); BM->addLoopMergeInst(Br->getFalseLabel()->getId(), // Merge Block BB->getId(), // Continue Target LoopControl, Parameters, SuccessorTrue); } else { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_unstructured_loop_controls)) { // For unstructured loop we add a special loop control instruction. // Simple example of unstructured loop is an infinite loop, that has // no terminate instruction. BM->addLoopControlINTELInst(LoopControl, Parameters, SuccessorTrue); } } } return mapValue(V, BM->addBranchInst(SuccessorTrue, BB)); } // For "do-while" (and in some cases, for "for" and "while") loops, // llvm.loop metadata is attached to a conditional branch instructions SPIRVLabel *SuccessorFalse = static_cast(transValue(Branch->getSuccessor(1), BB)); if (LoopControl != spv::LoopControlMaskNone) { Function *Fun = Branch->getFunction(); DominatorTree DomTree(*Fun); LoopInfo LI(DomTree); for (const auto *LoopObj : LI.getLoopsInPreorder()) { // Check whether SuccessorFalse or SuccessorTrue is the loop header BB. // For example consider following LLVM IR: // br i1 %compare, label %for.body, label %for.end // <- SuccessorTrue is 'for.body' aka successor(0) // br i1 %compare.not, label %for.end, label %for.body // <- SuccessorTrue is 'for.end' aka successor(1) // meanwhile the true successor (by definition) should be a loop header // aka 'for.body' if (LoopObj->getHeader() == Branch->getSuccessor(1)) // SuccessorFalse is the loop header BB. BM->addLoopMergeInst(SuccessorTrue->getId(), // Merge Block BB->getId(), // Continue Target LoopControl, Parameters, SuccessorFalse); else // SuccessorTrue is the loop header BB. BM->addLoopMergeInst(SuccessorFalse->getId(), // Merge Block BB->getId(), // Continue Target LoopControl, Parameters, SuccessorTrue); } } return mapValue( V, BM->addBranchConditionalInst(transValue(Branch->getCondition(), BB), SuccessorTrue, SuccessorFalse, BB)); } if (auto Phi = dyn_cast(V)) { std::vector IncomingPairs; for (size_t I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) { IncomingPairs.push_back(transValue(Phi->getIncomingValue(I), BB, true, FuncTransMode::Pointer)); IncomingPairs.push_back(transValue(Phi->getIncomingBlock(I), nullptr)); } return mapValue(V, BM->addPhiInst(transScavengedType(Phi), IncomingPairs, BB)); } if (auto Ext = dyn_cast(V)) { return mapValue(V, BM->addCompositeExtractInst( transType(Ext->getType()), transValue(Ext->getAggregateOperand(), BB), Ext->getIndices(), BB)); } if (auto Ins = dyn_cast(V)) { return mapValue(V, BM->addCompositeInsertInst( transValue(Ins->getInsertedValueOperand(), BB), transValue(Ins->getAggregateOperand(), BB), Ins->getIndices(), BB)); } if (UnaryInstruction *U = dyn_cast(V)) { if (auto *Init = unwrapSpecialTypeInitializer(U)) return mapValue(V, transValue(Init, BB)); auto UI = transUnaryInst(U, BB); return mapValue(V, UI ? UI : transValue(U->getOperand(0), BB)); } if (GetElementPtrInst *GEP = dyn_cast(V)) { std::vector Indices; for (unsigned I = 0, E = GEP->getNumIndices(); I != E; ++I) Indices.push_back(transValue(GEP->getOperand(I + 1), BB)); auto *PointerOperand = GEP->getPointerOperand(); auto *TransPointerOperand = transValue(PointerOperand, BB); // Certain array-related optimization hints can be expressed via // LLVM metadata. For the purpose of linking this metadata with // the accessed array variables, our GEP may have been marked into // a so-called index group, an MDNode by itself. if (MDNode *IndexGroup = GEP->getMetadata("llvm.index.group")) { SPIRVValue *ActualMemoryPtr = TransPointerOperand; if (auto *Load = dyn_cast(PointerOperand)) { ActualMemoryPtr = transValue(Load->getPointerOperand(), BB); } SPIRVId AccessedArrayId = ActualMemoryPtr->getId(); unsigned NumOperands = IndexGroup->getNumOperands(); // When we're working with embedded loops, it's natural that // the outer loop's hints apply to all code contained within. // The inner loop's specific hints, however, should stay private // to the inner loop's scope. // Consequently, the following division of the index group metadata // nodes emerges: // 1) The metadata node has no operands. It will be directly referenced // from within the optimization hint metadata. if (NumOperands == 0) IndexGroupArrayMap[IndexGroup].insert(AccessedArrayId); // 2) The metadata node has several operands. It serves to link an index // group specific to some embedded loop with other index groups that // mark the same array variable for the outer loop(s). for (unsigned I = 0; I < NumOperands; ++I) { auto *ContainedIndexGroup = getMDOperandAsMDNode(IndexGroup, I); IndexGroupArrayMap[ContainedIndexGroup].insert(AccessedArrayId); } } SPIRVType *TranslatedTy = transPointerType( GEP->getResultElementType(), GEP->getType()->getPointerAddressSpace()); return mapValue(V, BM->addPtrAccessChainInst(TranslatedTy, TransPointerOperand, Indices, BB, GEP->isInBounds())); } if (auto Ext = dyn_cast(V)) { auto Index = Ext->getIndexOperand(); if (auto Const = dyn_cast(Index)) return mapValue(V, BM->addCompositeExtractInst( transType(Ext->getType()), transValue(Ext->getVectorOperand(), BB), std::vector(1, Const->getZExtValue()), BB)); else return mapValue(V, BM->addVectorExtractDynamicInst( transValue(Ext->getVectorOperand(), BB), transValue(Index, BB), BB)); } if (auto Ins = dyn_cast(V)) { auto Index = Ins->getOperand(2); if (auto Const = dyn_cast(Index)) { return mapValue( V, BM->addCompositeInsertInst( transValue(Ins->getOperand(1), BB, true, FuncTransMode::Pointer), transValue(Ins->getOperand(0), BB), std::vector(1, Const->getZExtValue()), BB)); } else return mapValue( V, BM->addVectorInsertDynamicInst(transValue(Ins->getOperand(0), BB), transValue(Ins->getOperand(1), BB), transValue(Index, BB), BB)); } if (auto SF = dyn_cast(V)) { std::vector Comp; for (auto &I : SF->getShuffleMask()) Comp.push_back(I); return mapValue(V, BM->addVectorShuffleInst( transType(SF->getType()), transValue(SF->getOperand(0), BB), transValue(SF->getOperand(1), BB), Comp, BB)); } if (AtomicRMWInst *ARMW = dyn_cast(V)) { AtomicRMWInst::BinOp Op = ARMW->getOperation(); const bool SupportedAtomicInst = AtomicRMWInst::isFPOperation(Op) ? (Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub || Op == AtomicRMWInst::FMin || Op == AtomicRMWInst::FMax) : Op != AtomicRMWInst::Nand; if (!BM->getErrorLog().checkError( SupportedAtomicInst, SPIRVEC_InvalidInstruction, V, "Atomic " + AtomicRMWInst::getOperationName(Op).str() + " is not supported in SPIR-V!\n")) return nullptr; AtomicOrderingCABI Ordering = llvm::toCABI(ARMW->getOrdering()); auto MemSem = OCLMemOrderMap::map(static_cast(Ordering)); std::vector Operands(4); Operands[0] = ARMW->getPointerOperand(); // To get the memory scope argument we might use ARMW->getSyncScopeID(), but // atomicrmw LLVM instruction is not aware of OpenCL(or SPIR-V) memory scope // enumeration. And assuming the produced SPIR-V module will be consumed in // an OpenCL environment, we can use the same memory scope as OpenCL atomic // functions that don't have memory_scope argument i.e. memory_scope_device. // See the OpenCL C specification p6.13.11. "Atomic Functions" Operands[1] = getUInt32(M, spv::ScopeDevice); Operands[2] = getUInt32(M, MemSem); Operands[3] = ARMW->getValOperand(); std::vector OpVals = transValue(Operands, BB); std::vector Ops = BM->getIds(OpVals); SPIRVType *Ty = transType(ARMW->getType()); spv::Op OC; if (Op == AtomicRMWInst::FSub) { // Implement FSub through FNegate and AtomicFAddExt Ops[3] = BM->addUnaryInst(OpFNegate, Ty, OpVals[3], BB)->getId(); OC = OpAtomicFAddEXT; } else OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op); return mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty)); } if (IntrinsicInst *II = dyn_cast(V)) { SPIRVValue *BV = transIntrinsicInst(II, BB); return BV ? mapValue(V, BV) : nullptr; } if (FenceInst *FI = dyn_cast(V)) { SPIRVValue *BV = transFenceInst(FI, BB); return BV ? mapValue(V, BV) : nullptr; } if (InlineAsm *IA = dyn_cast(V)) if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_inline_assembly)) return mapValue(V, transAsmINTEL(IA)); if (CallInst *CI = dyn_cast(V)) { if (auto Alias = dyn_cast_or_null(CI->getCalledOperand())) { CI->setCalledFunction(cast(Alias->getAliasee())); } return mapValue(V, transCallInst(CI, BB)); } if (Instruction *Inst = dyn_cast(V)) { BM->SPIRVCK(false, InvalidInstruction, toString(Inst)); } llvm_unreachable("Not implemented"); return nullptr; } SPIRVType *LLVMToSPIRVBase::mapType(Type *T, SPIRVType *BT) { assert(!T->isPointerTy() && "Pointer types cannot be stored in the type map"); auto EmplaceStatus = TypeMap.try_emplace(T, BT); // TODO: Uncomment the assertion, once the type mapping issue is resolved // assert(EmplaceStatus.second && "The type was already added to the map"); SPIRVDBG(dbgs() << "[mapType] " << *T << " => "; spvdbgs() << *BT << '\n'); if (!EmplaceStatus.second) return TypeMap[T]; return BT; } SPIRVValue *LLVMToSPIRVBase::mapValue(Value *V, SPIRVValue *BV) { auto Loc = ValueMap.find(V); if (Loc != ValueMap.end()) { if (Loc->second == BV) return BV; assert(Loc->second->isForward() && "LLVM Value is mapped to different SPIRV Values"); auto Forward = static_cast(Loc->second); BM->replaceForward(Forward, BV); } ValueMap[V] = BV; SPIRVDBG(dbgs() << "[mapValue] " << *V << " => "; spvdbgs() << BV << "\n"); return BV; } bool LLVMToSPIRVBase::shouldTryToAddMemAliasingDecoration(Instruction *Inst) { // Limit translation of aliasing metadata with only this set of instructions // gracefully considering others as compilation mistakes and ignoring them if (!Inst->mayReadOrWriteMemory()) return false; // Loads and Stores are handled during memory access mask addition if (isa(Inst) || isa(Inst)) return false; CallInst *CI = dyn_cast(Inst); if (!CI) return true; if (Function *Fun = CI->getCalledFunction()) { // Calls to intrinsics are skipped. At some point lifetime start/end will be // handled separately, but specification isn't ready. if (Fun->isIntrinsic()) return false; // Also skip SPIR-V instructions that don't have result id to attach the // decorations if (isBuiltinTransToInst(Fun)) if (Fun->getReturnType()->isVoidTy()) return false; } return true; } void addFuncPointerCallArgumentAttributes(CallInst *CI, SPIRVValue *FuncPtrCall) { for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ++ArgNo) { for (const auto &I : CI->getAttributes().getParamAttrs(ArgNo)) { spv::FunctionParameterAttribute Attr = spv::FunctionParameterAttributeMax; SPIRSPIRVFuncParamAttrMap::find(I.getKindAsEnum(), &Attr); if (Attr != spv::FunctionParameterAttributeMax) FuncPtrCall->addDecorate( new SPIRVDecorate(spv::internal::DecorationArgumentAttributeINTEL, FuncPtrCall, ArgNo, Attr)); } } } #define ONE_STRING_DECORATION_CASE(NAME, NAMESPACE) \ case NAMESPACE::Decoration##NAME: { \ ErrLog.checkError(NumOperands == 2, SPIRVEC_InvalidLlvmModule, \ #NAME " requires exactly 1 extra operand"); \ auto *StrDecoEO = dyn_cast(DecoMD->getOperand(1)); \ ErrLog.checkError(StrDecoEO, SPIRVEC_InvalidLlvmModule, \ #NAME " requires extra operand to be a string"); \ Target->addDecorate( \ new SPIRVDecorate##NAME##Attr(Target, StrDecoEO->getString().str())); \ break; \ } #define ONE_INT_DECORATION_CASE(NAME, NAMESPACE, TYPE) \ case NAMESPACE::Decoration##NAME: { \ ErrLog.checkError(NumOperands == 2, SPIRVEC_InvalidLlvmModule, \ #NAME " requires exactly 1 extra operand"); \ auto *IntDecoEO = \ mdconst::dyn_extract(DecoMD->getOperand(1)); \ ErrLog.checkError(IntDecoEO, SPIRVEC_InvalidLlvmModule, \ #NAME " requires extra operand to be an integer"); \ Target->addDecorate(new SPIRVDecorate##NAME( \ Target, static_cast(IntDecoEO->getZExtValue()))); \ break; \ } #define TWO_INT_DECORATION_CASE(NAME, NAMESPACE, TYPE1, TYPE2) \ case NAMESPACE::Decoration##NAME: { \ ErrLog.checkError(NumOperands == 3, SPIRVEC_InvalidLlvmModule, \ #NAME " requires exactly 2 extra operands"); \ auto *IntDecoEO1 = \ mdconst::dyn_extract(DecoMD->getOperand(1)); \ ErrLog.checkError(IntDecoEO1, SPIRVEC_InvalidLlvmModule, \ #NAME " requires first extra operand to be an integer"); \ auto *IntDecoEO2 = \ mdconst::dyn_extract(DecoMD->getOperand(2)); \ ErrLog.checkError(IntDecoEO2, SPIRVEC_InvalidLlvmModule, \ #NAME \ " requires second extra operand to be an integer"); \ Target->addDecorate(new SPIRVDecorate##NAME( \ Target, static_cast(IntDecoEO1->getZExtValue()), \ static_cast(IntDecoEO2->getZExtValue()))); \ break; \ } void checkIsGlobalVar(SPIRVEntry *E, Decoration Dec) { std::string ErrStr = SPIRVDecorationNameMap::map(Dec) + " can only be applied to a variable"; E->getErrorLog().checkError(E->isVariable(), SPIRVEC_InvalidModule, ErrStr); auto AddrSpace = SPIRSPIRVAddrSpaceMap::rmap( static_cast(E)->getStorageClass()); ErrStr += " in a global (module) scope"; E->getErrorLog().checkError(AddrSpace == SPIRAS_Global, SPIRVEC_InvalidModule, ErrStr); } static void transMetadataDecorations(Metadata *MD, SPIRVValue *Target) { SPIRVErrorLog &ErrLog = Target->getErrorLog(); auto *ArgDecoMD = dyn_cast(MD); assert(ArgDecoMD && "Decoration list must be a metadata node"); for (unsigned I = 0, E = ArgDecoMD->getNumOperands(); I != E; ++I) { auto *DecoMD = dyn_cast(ArgDecoMD->getOperand(I)); ErrLog.checkError(DecoMD, SPIRVEC_InvalidLlvmModule, "Decoration does not name metadata"); ErrLog.checkError(DecoMD->getNumOperands() > 0, SPIRVEC_InvalidLlvmModule, "Decoration metadata must have at least one operand"); auto *DecoKindConst = mdconst::dyn_extract(DecoMD->getOperand(0)); ErrLog.checkError(DecoKindConst, SPIRVEC_InvalidLlvmModule, "First operand of decoration must be the kind"); auto DecoKind = static_cast(DecoKindConst->getZExtValue()); const size_t NumOperands = DecoMD->getNumOperands(); switch (static_cast(DecoKind)) { case DecorationAlignment: { // Handle Alignment via SPIRVValue::setAlignment() to avoid duplicate // Alignment decorations. auto *Alignment = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError(Alignment, SPIRVEC_InvalidLlvmModule, "Alignment operand must be an integer."); Target->setAlignment(Alignment->getZExtValue()); break; } ONE_STRING_DECORATION_CASE(MemoryINTEL, spv) ONE_STRING_DECORATION_CASE(UserSemantic, spv) ONE_INT_DECORATION_CASE(AliasScopeINTEL, spv, SPIRVId) ONE_INT_DECORATION_CASE(NoAliasINTEL, spv, SPIRVId) ONE_INT_DECORATION_CASE(InitiationIntervalINTEL, spv::internal, SPIRVWord) ONE_INT_DECORATION_CASE(MaxConcurrencyINTEL, spv::internal, SPIRVWord) ONE_INT_DECORATION_CASE(PipelineEnableINTEL, spv::internal, SPIRVWord) TWO_INT_DECORATION_CASE(FunctionRoundingModeINTEL, spv, SPIRVWord, FPRoundingMode); TWO_INT_DECORATION_CASE(FunctionDenormModeINTEL, spv, SPIRVWord, FPDenormMode); TWO_INT_DECORATION_CASE(FunctionFloatingPointModeINTEL, spv, SPIRVWord, FPOperationMode); TWO_INT_DECORATION_CASE(FuseLoopsInFunctionINTEL, spv, SPIRVWord, SPIRVWord); TWO_INT_DECORATION_CASE(MathOpDSPModeINTEL, spv::internal, SPIRVWord, SPIRVWord); case DecorationStallEnableINTEL: { Target->addDecorate(new SPIRVDecorateStallEnableINTEL(Target)); break; } case DecorationMergeINTEL: { ErrLog.checkError(NumOperands == 3, SPIRVEC_InvalidLlvmModule, "MergeINTEL requires exactly 3 extra operands"); auto *Name = dyn_cast(DecoMD->getOperand(1)); ErrLog.checkError( Name, SPIRVEC_InvalidLlvmModule, "MergeINTEL requires first extra operand to be a string"); auto *Direction = dyn_cast(DecoMD->getOperand(2)); ErrLog.checkError( Direction, SPIRVEC_InvalidLlvmModule, "MergeINTEL requires second extra operand to be a string"); Target->addDecorate(new SPIRVDecorateMergeINTELAttr( Target, Name->getString().str(), Direction->getString().str())); break; } case DecorationLinkageAttributes: { ErrLog.checkError(NumOperands == 3, SPIRVEC_InvalidLlvmModule, "LinkageAttributes requires exactly 3 extra operands"); auto *Name = dyn_cast(DecoMD->getOperand(1)); ErrLog.checkError( Name, SPIRVEC_InvalidLlvmModule, "LinkageAttributes requires first extra operand to be a string"); auto *Type = mdconst::dyn_extract(DecoMD->getOperand(2)); ErrLog.checkError( Type, SPIRVEC_InvalidLlvmModule, "LinkageAttributes requires second extra operand to be an int"); auto TypeKind = static_cast(Type->getZExtValue()); Target->addDecorate(new SPIRVDecorateLinkageAttr( Target, Name->getString().str(), TypeKind)); break; } case spv::internal::DecorationHostAccessINTEL: case DecorationHostAccessINTEL: { checkIsGlobalVar(Target, DecoKind); ErrLog.checkError(NumOperands == 3, SPIRVEC_InvalidLlvmModule, "HostAccessINTEL requires exactly 2 extra operands " "after the decoration kind number"); auto *AccessMode = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError( AccessMode, SPIRVEC_InvalidLlvmModule, "HostAccessINTEL requires first extra operand to be an int"); const HostAccessQualifier Q = static_cast(AccessMode->getZExtValue()); auto *Name = dyn_cast(DecoMD->getOperand(2)); ErrLog.checkError( Name, SPIRVEC_InvalidLlvmModule, "HostAccessINTEL requires second extra operand to be a string"); if (DecoKind == DecorationHostAccessINTEL) Target->addDecorate(new SPIRVDecorateHostAccessINTEL( Target, Q, Name->getString().str())); else Target->addDecorate(new SPIRVDecorateHostAccessINTELLegacy( Target, Q, Name->getString().str())); break; } case spv::internal::DecorationInitModeINTEL: case DecorationInitModeINTEL: { checkIsGlobalVar(Target, DecoKind); ErrLog.checkError(static_cast(Target)->getInitializer(), SPIRVEC_InvalidLlvmModule, "InitModeINTEL only be applied to a global (module " "scope) variable which has an Initializer operand"); ErrLog.checkError(NumOperands == 2, SPIRVEC_InvalidLlvmModule, "InitModeINTEL requires exactly 1 extra operand"); auto *Trigger = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError(Trigger, SPIRVEC_InvalidLlvmModule, "InitModeINTEL requires extra operand to be an int"); const InitializationModeQualifier Q = static_cast(Trigger->getZExtValue()); if (DecoKind == DecorationInitModeINTEL) Target->addDecorate(new SPIRVDecorateInitModeINTEL(Target, Q)); else Target->addDecorate(new SPIRVDecorateInitModeINTELLegacy(Target, Q)); break; } case spv::internal::DecorationImplementInCSRINTEL: { checkIsGlobalVar(Target, DecoKind); ErrLog.checkError(NumOperands == 2, SPIRVEC_InvalidLlvmModule, "ImplementInCSRINTEL requires exactly 1 extra operand"); auto *Value = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError( Value, SPIRVEC_InvalidLlvmModule, "ImplementInCSRINTEL requires extra operand to be an integer"); Target->addDecorate( new SPIRVDecorateImplementInCSRINTEL(Target, Value->getZExtValue())); break; } case DecorationImplementInRegisterMapINTEL: { checkIsGlobalVar(Target, DecoKind); ErrLog.checkError( NumOperands == 2, SPIRVEC_InvalidLlvmModule, "ImplementInRegisterMapINTEL requires exactly 1 extra operand"); auto *Value = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError(Value, SPIRVEC_InvalidLlvmModule, "ImplementInRegisterMapINTEL requires extra operand to " "be an integer"); Target->addDecorate(new SPIRVDecorateImplementInRegisterMapINTEL( Target, Value->getZExtValue())); break; } case DecorationCacheControlLoadINTEL: { ErrLog.checkError( NumOperands == 3, SPIRVEC_InvalidLlvmModule, "CacheControlLoadINTEL requires exactly 2 extra operands"); auto *CacheLevel = mdconst::dyn_extract(DecoMD->getOperand(1)); auto *CacheControl = mdconst::dyn_extract(DecoMD->getOperand(2)); ErrLog.checkError(CacheLevel, SPIRVEC_InvalidLlvmModule, "CacheControlLoadINTEL cache level operand is required " "to be an integer"); ErrLog.checkError(CacheControl, SPIRVEC_InvalidLlvmModule, "CacheControlLoadINTEL cache control operand is " "required to be an integer"); Target->addDecorate(new SPIRVDecorateCacheControlLoadINTEL( Target, CacheLevel->getZExtValue(), static_cast(CacheControl->getZExtValue()))); break; } case DecorationCacheControlStoreINTEL: { ErrLog.checkError( NumOperands == 3, SPIRVEC_InvalidLlvmModule, "CacheControlStoreINTEL requires exactly 2 extra operands"); auto *CacheLevel = mdconst::dyn_extract(DecoMD->getOperand(1)); auto *CacheControl = mdconst::dyn_extract(DecoMD->getOperand(2)); ErrLog.checkError(CacheLevel, SPIRVEC_InvalidLlvmModule, "CacheControlStoreINTEL cache level operand is " "required to be an integer"); ErrLog.checkError(CacheControl, SPIRVEC_InvalidLlvmModule, "CacheControlStoreINTEL cache control operand is " "required to be an integer"); Target->addDecorate(new SPIRVDecorateCacheControlStoreINTEL( Target, CacheLevel->getZExtValue(), static_cast(CacheControl->getZExtValue()))); break; } default: { if (NumOperands == 1) { Target->addDecorate(new SPIRVDecorate(DecoKind, Target)); break; } auto *DecoValEO1 = mdconst::dyn_extract(DecoMD->getOperand(1)); ErrLog.checkError( DecoValEO1, SPIRVEC_InvalidLlvmModule, "First extra operand in default decoration case must be integer."); if (NumOperands == 2) { Target->addDecorate( new SPIRVDecorate(DecoKind, Target, DecoValEO1->getZExtValue())); break; } auto *DecoValEO2 = mdconst::dyn_extract(DecoMD->getOperand(2)); ErrLog.checkError( DecoValEO2, SPIRVEC_InvalidLlvmModule, "Second extra operand in default decoration case must be integer."); ErrLog.checkError(NumOperands == 3, SPIRVEC_InvalidLlvmModule, "At most 2 extra operands expected."); Target->addDecorate(new SPIRVDecorate(DecoKind, Target, DecoValEO1->getZExtValue(), DecoValEO2->getZExtValue())); } } } } #undef ONE_STRING_DECORATION_CASE #undef ONE_INT_DECORATION_CASE #undef TWO_INT_DECORATION_CASE bool LLVMToSPIRVBase::transDecoration(Value *V, SPIRVValue *BV) { if (!transAlign(V, BV)) return false; if ((isa(V) && cast(V)->isVolatile()) || (isa(V) && cast(V)->isVolatile())) BV->setVolatile(true); if (auto BVO = dyn_cast_or_null(V)) { if (BVO->hasNoSignedWrap()) { BV->setNoIntegerDecorationWrap(true); } if (BVO->hasNoUnsignedWrap()) { BV->setNoIntegerDecorationWrap(true); } } if (auto BVF = dyn_cast_or_null(V)) { auto Opcode = BVF->getOpcode(); if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub || Opcode == Instruction::FMul || Opcode == Instruction::FDiv || Opcode == Instruction::FRem || ((Opcode == Instruction::FNeg || Opcode == Instruction::FCmp || BV->isExtInst()) && BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_6))) { FastMathFlags FMF = BVF->getFastMathFlags(); SPIRVWord M{0}; if (FMF.isFast()) M |= FPFastMathModeFastMask; else { if (FMF.noNaNs()) M |= FPFastMathModeNotNaNMask; if (FMF.noInfs()) M |= FPFastMathModeNotInfMask; if (FMF.noSignedZeros()) M |= FPFastMathModeNSZMask; if (FMF.allowReciprocal()) M |= FPFastMathModeAllowRecipMask; if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fp_fast_math_mode)) { if (FMF.allowContract()) { M |= FPFastMathModeAllowContractFastINTELMask; BM->addCapability(CapabilityFPFastMathModeINTEL); } if (FMF.allowReassoc()) { M |= FPFastMathModeAllowReassocINTELMask; BM->addCapability(CapabilityFPFastMathModeINTEL); } } } if (M != 0) { BV->setFPFastMathMode(M); if (Opcode == Instruction::FNeg || Opcode == Instruction::FCmp || BV->isExtInst()) BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_6); } } } if (Instruction *Inst = dyn_cast(V)) { if (shouldTryToAddMemAliasingDecoration(Inst)) transMemAliasingINTELDecorations(Inst, BV); if (auto *IDecoMD = Inst->getMetadata(SPIRV_MD_DECORATIONS)) transMetadataDecorations(IDecoMD, BV); } if (auto *CI = dyn_cast(V)) { auto OC = BV->getOpCode(); if (OC == OpSpecConstantTrue || OC == OpSpecConstantFalse || OC == OpSpecConstant) { auto SpecId = cast(CI->getArgOperand(0))->getZExtValue(); BV->addDecorate(DecorationSpecId, SpecId); } if (OC == OpFunctionPointerCallINTEL) addFuncPointerCallArgumentAttributes(CI, BV); } if (auto *GV = dyn_cast(V)) if (auto *GVDecoMD = GV->getMetadata(SPIRV_MD_DECORATIONS)) transMetadataDecorations(GVDecoMD, BV); return true; } bool LLVMToSPIRVBase::transAlign(Value *V, SPIRVValue *BV) { if (auto AL = dyn_cast(V)) { BM->setAlignment(BV, AL->getAlign().value()); return true; } if (auto GV = dyn_cast(V)) { BM->setAlignment(BV, GV->getAlignment()); return true; } return true; } // Apply aliasing decorations to instructions annotated with aliasing metadata. // Do it for any instruction but loads and stores. void LLVMToSPIRVBase::transMemAliasingINTELDecorations(Instruction *Inst, SPIRVValue *BV) { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_memory_access_aliasing)) return; if (MDNode *AliasingListMD = Inst->getMetadata(LLVMContext::MD_alias_scope)) { auto *MemAliasList = addMemAliasingINTELInstructions(BM, AliasingListMD); if (!MemAliasList) return; BV->addDecorate(new SPIRVDecorateId(DecorationAliasScopeINTEL, BV, MemAliasList->getId())); } if (MDNode *AliasingListMD = Inst->getMetadata(LLVMContext::MD_noalias)) { auto *MemAliasList = addMemAliasingINTELInstructions(BM, AliasingListMD); if (!MemAliasList) return; BV->addDecorate( new SPIRVDecorateId(DecorationNoAliasINTEL, BV, MemAliasList->getId())); } } /// Do this after source language is set. bool LLVMToSPIRVBase::transBuiltinSet() { SPIRVId EISId; if (!BM->importBuiltinSet("OpenCL.std", &EISId)) return false; if (SPIRVMDWalker(*M).getNamedMD("llvm.dbg.cu")) { if (!BM->importBuiltinSet( SPIRVBuiltinSetNameMap::map(BM->getDebugInfoEIS()), &EISId)) return false; } if (BM->preserveAuxData()) { if (!BM->importBuiltinSet( SPIRVBuiltinSetNameMap::map(SPIRVEIS_NonSemantic_AuxData), &EISId)) return false; } return true; } /// Translate sampler* spcv.cast(i32 arg) or /// sampler* __translate_sampler_initializer(i32 arg) /// Three cases are possible: /// arg = ConstantInt x -> SPIRVConstantSampler /// arg = i32 argument -> transValue(arg) /// arg = load from sampler -> look through load SPIRVValue *LLVMToSPIRVBase::oclTransSpvcCastSampler(CallInst *CI, SPIRVBasicBlock *BB) { assert(CI->getCalledFunction() && "Unexpected indirect call"); llvm::Function *F = CI->getCalledFunction(); auto FT = F->getFunctionType(); auto RT = FT->getReturnType(); assert(FT->getNumParams() == 1); if (!RT->isOpaquePointerTy()) { StructType *ST = dyn_cast(RT->getNonOpaquePointerElementType()); (void)ST; assert(isSPIRVStructType(ST, kSPIRVTypeName::Sampler) || (ST->isOpaque() && ST->getName() == kSPR2TypeName::Sampler)); } assert(FT->getParamType(0)->isIntegerTy() && "Invalid sampler type"); auto Arg = CI->getArgOperand(0); auto *TransRT = transPointerType(getOrCreateOpaqueStructType(M, kSPR2TypeName::Sampler), RT->getPointerAddressSpace()); auto GetSamplerConstant = [&](uint64_t SamplerValue) { auto AddrMode = (SamplerValue & 0xE) >> 1; auto Param = SamplerValue & 0x1; auto Filter = SamplerValue ? ((SamplerValue & 0x30) >> 4) - 1 : 0; auto *BV = BM->addSamplerConstant(TransRT, AddrMode, Param, Filter); return BV; }; if (auto Const = dyn_cast(Arg)) { // Sampler is declared as a kernel scope constant return GetSamplerConstant(Const->getZExtValue()); } else if (auto Load = dyn_cast(Arg)) { // If value of the sampler is loaded from a global constant, use its // initializer for initialization of the sampler. auto Op = Load->getPointerOperand(); assert(isa(Op) && "Unknown sampler pattern!"); auto GV = cast(Op); assert(GV->isConstant() || GV->getType()->getPointerAddressSpace() == SPIRAS_Constant); auto Initializer = GV->getInitializer(); assert(isa(Initializer) && "sampler not constant int?"); return GetSamplerConstant(cast(Initializer)->getZExtValue()); } // Sampler is a function argument auto BV = transValue(Arg, BB); assert(BV && BV->getType() == TransRT); return BV; } using DecorationsInfoVec = std::vector>>; struct AnnotationDecorations { DecorationsInfoVec MemoryAttributesVec; DecorationsInfoVec MemoryAccessesVec; }; struct IntelLSUControlsInfo { void setWithBitMask(unsigned ParamsBitMask) { if (ParamsBitMask & IntelFPGAMemoryAccessesVal::BurstCoalesce) BurstCoalesce = true; if (ParamsBitMask & IntelFPGAMemoryAccessesVal::CacheSizeFlag) CacheSizeInfo = 0; if (ParamsBitMask & IntelFPGAMemoryAccessesVal::DontStaticallyCoalesce) DontStaticallyCoalesce = true; if (ParamsBitMask & IntelFPGAMemoryAccessesVal::PrefetchFlag) PrefetchInfo = 0; } DecorationsInfoVec getDecorationsFromCurrentState() { DecorationsInfoVec ResultVec; // Simple flags if (BurstCoalesce) ResultVec.emplace_back(DecorationBurstCoalesceINTEL, std::vector()); if (DontStaticallyCoalesce) ResultVec.emplace_back(DecorationDontStaticallyCoalesceINTEL, std::vector()); // Conditional values if (CacheSizeInfo.hasValue()) { ResultVec.emplace_back( DecorationCacheSizeINTEL, std::vector{std::to_string(CacheSizeInfo.getValue())}); } if (PrefetchInfo.hasValue()) { ResultVec.emplace_back( DecorationPrefetchINTEL, std::vector{std::to_string(PrefetchInfo.getValue())}); } return ResultVec; } bool BurstCoalesce = false; llvm::Optional CacheSizeInfo; bool DontStaticallyCoalesce = false; llvm::Optional PrefetchInfo; }; // Handle optional var/ptr/global annotation parameter. It can be for example // { %struct.S, i8*, void ()* } { %struct.S undef, i8* null, // void ()* @_Z4blahv } // Now we will just handle integer constants (wrapped in a constant // struct, that is being bitcasted to i8*), converting them to string. // TODO: remove this workaround when/if an extension spec that allows or adds // variadic-arguments UserSemantic decoration void processOptionalAnnotationInfo(Constant *Const, std::string &AnnotationString) { if (!Const->getNumOperands()) return; if (auto *CStruct = dyn_cast(Const->getOperand(0))) { uint32_t NumOperands = CStruct->getNumOperands(); if (!NumOperands) return; if (auto *CInt = dyn_cast(CStruct->getOperand(0))) { AnnotationString += ": "; AnnotationString += std::to_string(CInt->getSExtValue()); } for (uint32_t I = 1; I != NumOperands; ++I) { if (auto *CInt = dyn_cast(CStruct->getOperand(I))) { AnnotationString += ", "; AnnotationString += std::to_string(CInt->getSExtValue()); } } } else if (auto *ZeroStruct = dyn_cast(Const->getOperand(0))) { // It covers case when all elements of struct are 0 and they become // zeroinitializer. It represents like: { i32 i32 ... } zeroinitializer uint32_t NumOperands = ZeroStruct->getType()->getStructNumElements(); AnnotationString += ": "; AnnotationString += "0"; for (uint32_t I = 1; I != NumOperands; ++I) { AnnotationString += ", "; AnnotationString += "0"; } } } // Process main var/ptr/global annotation string with the attached optional // integer parameters void processAnnotationString(IntrinsicInst *II, std::string &AnnotationString) { if (auto *GEP = dyn_cast(II->getArgOperand(1))) { if (auto *C = dyn_cast(GEP->getOperand(0))) { StringRef StrRef; getConstantStringInfo(C, StrRef); AnnotationString += StrRef.str(); } } if (auto *Cast = dyn_cast(II->getArgOperand(4))) if (auto *C = dyn_cast_or_null(Cast->getOperand(0))) processOptionalAnnotationInfo(C, AnnotationString); } // Try to parse the annotation decoration values in a string. These values must // be separated by a "," and must be either a word (including numbers) or a // quotation mark enclosed string. static bool tryParseAnnotationDecoValues(StringRef ValueStr, std::vector &ParsedArgs) { unsigned ValueStart = 0; bool IsParsingStringLiteral = false; for (unsigned I = 0; I < ValueStr.size(); ++I) { const char CurrentC = ValueStr[I]; if (IsParsingStringLiteral) { if (CurrentC == '"') { // We have reached the end of a string literal and have the arg string // between this character and the start of the string literal. IsParsingStringLiteral = false; ParsedArgs.push_back(ValueStr.substr(ValueStart, I - ValueStart).str()); // End of a string literal must either be at the end of the values or // right before a comma. if (I + 1 != ValueStr.size() && ValueStr[I + 1] != ',') return false; // Skip the , delimiter and go directly to the start of next value. ValueStart = (++I) + 1; } continue; } if (CurrentC == ',') { // Since we are not currently in a string literal, comma denotes a // separation of decoration arguments and we can copy the substring we are // currently parsing. ParsedArgs.push_back(ValueStr.substr(ValueStart, I - ValueStart).str()); ValueStart = I + 1; continue; } if (CurrentC == '"') { // We are entering a string literal. This must be either at the beginning // of the values or right after a comma. if (I != 0 && ValueStr[I - 1] != ',') return false; IsParsingStringLiteral = true; ValueStart = I + 1; continue; } // Any other character will be consumed as part of the argument. } // If we were still parsing a decoration argument when reaching the end of the // parsed string, we must be at the end of the argument. if (ValueStart < ValueStr.size()) ParsedArgs.push_back( ValueStr.substr(ValueStart, ValueStr.size() - ValueStart).str()); // At the end, the arguments parsed are valid if we were not parsing a string // literal with no end. return !IsParsingStringLiteral; } AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM, StringRef AnnotatedCode) { AnnotationDecorations Decorates; // Annotation string decorations are separated into {word} OR // {word:value,value,...} blocks, where value is either a word (including // numbers) or a quotation mark enclosed string. std::regex DecorationRegex("\\{\\w([\\w:,-]|\"[^\"]*\")*\\}"); using RegexIterT = std::regex_iterator; RegexIterT DecorationsIt(AnnotatedCode.begin(), AnnotatedCode.end(), DecorationRegex); RegexIterT DecorationsEnd; // If we didn't find any annotations that are separated as described above, // then add a UserSemantic decoration if (DecorationsIt == DecorationsEnd) { Decorates.MemoryAttributesVec.emplace_back( DecorationUserSemantic, std::vector{AnnotatedCode.str()}); return Decorates; } const bool AllowFPGAMemAccesses = BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_memory_accesses); const bool AllowFPGAMemAttr = BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes); bool ValidDecorationFound = false; DecorationsInfoVec DecorationsVec; IntelLSUControlsInfo LSUControls; for (; DecorationsIt != DecorationsEnd; ++DecorationsIt) { // Drop the braces surrounding the actual decoration const StringRef AnnotatedDecoration = AnnotatedCode.substr( DecorationsIt->position() + 1, DecorationsIt->length() - 2); std::pair Split = AnnotatedDecoration.split(':'); StringRef Name = Split.first, ValueStr = Split.second; unsigned DecorationKind = 0; if (!Name.getAsInteger(10, DecorationKind)) { // If the name is a number it represents the decoration by its kind. std::vector DecValues; if (tryParseAnnotationDecoValues(ValueStr, DecValues)) { ValidDecorationFound = true; DecorationsVec.emplace_back(static_cast(DecorationKind), std::move(DecValues)); } continue; } if (AllowFPGAMemAccesses) { if (Name == "params") { ValidDecorationFound = true; unsigned ParamsBitMask = 0; bool Failure = ValueStr.getAsInteger(10, ParamsBitMask); assert(!Failure && "Non-integer LSU controls value"); (void)Failure; LSUControls.setWithBitMask(ParamsBitMask); } else if (Name == "cache-size") { ValidDecorationFound = true; if (!LSUControls.CacheSizeInfo.hasValue()) continue; unsigned CacheSizeValue = 0; bool Failure = ValueStr.getAsInteger(10, CacheSizeValue); assert(!Failure && "Non-integer cache size value"); (void)Failure; LSUControls.CacheSizeInfo = CacheSizeValue; } // TODO: Support LSU prefetch size, which currently defaults to 0 } if (AllowFPGAMemAttr) { std::vector DecValues; Decoration Dec; if (Name == "pump") { ValidDecorationFound = true; Dec = llvm::StringSwitch(ValueStr) .Case("1", DecorationSinglepumpINTEL) .Case("2", DecorationDoublepumpINTEL); } else if (Name == "register") { ValidDecorationFound = true; Dec = DecorationRegisterINTEL; } else if (Name == "simple_dual_port") { ValidDecorationFound = true; Dec = DecorationSimpleDualPortINTEL; } else { Dec = llvm::StringSwitch(Name) .Case("memory", DecorationMemoryINTEL) .Case("numbanks", DecorationNumbanksINTEL) .Case("bankwidth", DecorationBankwidthINTEL) .Case("private_copies", DecorationMaxPrivateCopiesINTEL) .Case("max_replicates", DecorationMaxReplicatesINTEL) .Case("bank_bits", DecorationBankBitsINTEL) .Case("merge", DecorationMergeINTEL) .Case("force_pow2_depth", DecorationForcePow2DepthINTEL) .Default(DecorationUserSemantic); if (Dec == DecorationUserSemantic) DecValues = std::vector({AnnotatedDecoration.str()}); else if (Dec == DecorationMergeINTEL) { ValidDecorationFound = true; std::pair MergeValues = ValueStr.split(':'); DecValues = std::vector( {MergeValues.first.str(), MergeValues.second.str()}); } else if (Dec == DecorationBankBitsINTEL) { ValidDecorationFound = true; SmallVector BitsStrs; ValueStr.split(BitsStrs, ','); DecValues.reserve(BitsStrs.size()); for (const StringRef &BitsStr : BitsStrs) DecValues.push_back(BitsStr.str()); } else { ValidDecorationFound = true; DecValues = std::vector({ValueStr.str()}); } } DecorationsVec.emplace_back(Dec, std::move(DecValues)); } } // Even if there is an annotation string that is split in blocks like Intel // FPGA annotation, it's not necessarily an FPGA annotation. Translate the // whole string as UserSemantic decoration in this case. if (ValidDecorationFound) Decorates.MemoryAttributesVec = DecorationsVec; else Decorates.MemoryAttributesVec.emplace_back( DecorationUserSemantic, std::vector({AnnotatedCode.str()})); Decorates.MemoryAccessesVec = LSUControls.getDecorationsFromCurrentState(); return Decorates; } std::vector getBankBitsFromStrings(const std::vector &BitsStrings) { std::vector Bits(BitsStrings.size()); for (size_t J = 0; J < BitsStrings.size(); ++J) if (StringRef(BitsStrings[J]).getAsInteger(10, Bits[J])) return {}; return Bits; } void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) { SPIRVModule *M = E->getModule(); for (const auto &I : Decorations) { // Such decoration already exists on a type, skip it if (E->hasDecorate(I.first, /*Index=*/0, /*Result=*/nullptr)) { continue; } switch (I.first) { case DecorationUserSemantic: M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "UserSemantic requires a single argument."); E->addDecorate(new SPIRVDecorateUserSemanticAttr(E, I.second[0])); break; case DecorationMemoryINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes)) { M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "MemoryINTEL requires a single argument."); E->addDecorate(new SPIRVDecorateMemoryINTELAttr(E, I.second[0])); } } break; case DecorationMergeINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes)) { M->getErrorLog().checkError(I.second.size() == 2, SPIRVEC_InvalidLlvmModule, "MergeINTEL requires two arguments."); // First argument is the name and the second argument is the direction. E->addDecorate( new SPIRVDecorateMergeINTELAttr(E, I.second[0], I.second[1])); } } break; case DecorationBankBitsINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes)) { M->getErrorLog().checkError( I.second.size() > 0, SPIRVEC_InvalidLlvmModule, "BankBitsINTEL requires at least one argument."); E->addDecorate(new SPIRVDecorateBankBitsINTELAttr( E, getBankBitsFromStrings(I.second))); } } break; case DecorationRegisterINTEL: case DecorationSinglepumpINTEL: case DecorationDoublepumpINTEL: case DecorationSimpleDualPortINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes)) { M->getErrorLog().checkError(I.second.empty(), SPIRVEC_InvalidLlvmModule, "Decoration takes no arguments."); E->addDecorate(I.first); } } break; case DecorationBurstCoalesceINTEL: case DecorationDontStaticallyCoalesceINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_accesses)) { M->getErrorLog().checkError(I.second.empty(), SPIRVEC_InvalidLlvmModule, "Decoration takes no arguments."); E->addDecorate(I.first); } } break; case DecorationNumbanksINTEL: case DecorationBankwidthINTEL: case DecorationMaxPrivateCopiesINTEL: case DecorationMaxReplicatesINTEL: case DecorationForcePow2DepthINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_attributes)) { M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "Decoration requires a single argument."); SPIRVWord Result = 0; StringRef(I.second[0]).getAsInteger(10, Result); E->addDecorate(I.first, Result); } } break; case DecorationCacheSizeINTEL: case DecorationPrefetchINTEL: { if (M->isAllowedToUseExtension( ExtensionID::SPV_INTEL_fpga_memory_accesses)) { M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "Decoration requires a single argument."); SPIRVWord Result = 0; StringRef(I.second[0]).getAsInteger(10, Result); E->addDecorate(I.first, Result); } } break; default: // Other decorations are either not supported by the translator or // handled in other places. break; } } } void addAnnotationDecorationsForStructMember(SPIRVEntry *E, SPIRVWord MemberNumber, DecorationsInfoVec &Decorations) { SPIRVModule *M = E->getModule(); for (const auto &I : Decorations) { // Such decoration already exists on a type, skip it if (E->hasMemberDecorate(I.first, /*Index=*/0, MemberNumber, /*Result=*/nullptr)) { continue; } switch (I.first) { case DecorationUserSemantic: M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "UserSemantic requires a single argument."); E->addMemberDecorate(new SPIRVMemberDecorateUserSemanticAttr( E, MemberNumber, I.second[0])); break; case DecorationMemoryINTEL: M->getErrorLog().checkError(I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "MemoryINTEL requires a single argument."); E->addMemberDecorate( new SPIRVMemberDecorateMemoryINTELAttr(E, MemberNumber, I.second[0])); break; case DecorationMergeINTEL: { M->getErrorLog().checkError(I.second.size() == 2, SPIRVEC_InvalidLlvmModule, "MergeINTEL requires two arguments."); // First argument is the name, the other is the direction. E->addMemberDecorate(new SPIRVMemberDecorateMergeINTELAttr( E, MemberNumber, I.second[0], I.second[1])); } break; case DecorationBankBitsINTEL: M->getErrorLog().checkError( I.second.size() > 0, SPIRVEC_InvalidLlvmModule, "BankBitsINTEL requires at least one argument."); E->addMemberDecorate(new SPIRVMemberDecorateBankBitsINTELAttr( E, MemberNumber, getBankBitsFromStrings(I.second))); break; case DecorationRegisterINTEL: case DecorationSinglepumpINTEL: case DecorationDoublepumpINTEL: case DecorationSimpleDualPortINTEL: M->getErrorLog().checkError(I.second.empty(), SPIRVEC_InvalidLlvmModule, "Member decoration takes no arguments."); E->addMemberDecorate(MemberNumber, I.first); break; // The rest of IntelFPGA decorations: // DecorationNumbanksINTEL // DecorationBankwidthINTEL // DecorationMaxPrivateCopiesINTEL // DecorationMaxReplicatesINTEL // DecorationForcePow2DepthINTEL default: M->getErrorLog().checkError( I.second.size() == 1, SPIRVEC_InvalidLlvmModule, "Member decoration requires a single argument."); SPIRVWord Result = 0; StringRef(I.second[0]).getAsInteger(10, Result); E->addMemberDecorate(MemberNumber, I.first, Result); break; } } } bool LLVMToSPIRVBase::isKnownIntrinsic(Intrinsic::ID Id) { // Known intrinsics usually do not need translation of their declaration switch (Id) { case Intrinsic::abs: case Intrinsic::assume: case Intrinsic::bitreverse: case Intrinsic::ceil: case Intrinsic::copysign: case Intrinsic::cos: case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::fabs: case Intrinsic::floor: case Intrinsic::fma: case Intrinsic::log: case Intrinsic::log10: case Intrinsic::log2: case Intrinsic::maximum: case Intrinsic::maxnum: case Intrinsic::smax: case Intrinsic::umax: case Intrinsic::minimum: case Intrinsic::minnum: case Intrinsic::smin: case Intrinsic::umin: case Intrinsic::nearbyint: case Intrinsic::pow: case Intrinsic::powi: case Intrinsic::rint: case Intrinsic::round: case Intrinsic::roundeven: case Intrinsic::sin: case Intrinsic::sqrt: case Intrinsic::trunc: case Intrinsic::ctpop: case Intrinsic::ctlz: case Intrinsic::cttz: case Intrinsic::expect: case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::experimental_constrained_fadd: case Intrinsic::experimental_constrained_fsub: case Intrinsic::experimental_constrained_fmul: case Intrinsic::experimental_constrained_fdiv: case Intrinsic::experimental_constrained_frem: case Intrinsic::experimental_constrained_fma: case Intrinsic::experimental_constrained_fptoui: case Intrinsic::experimental_constrained_fptosi: case Intrinsic::experimental_constrained_uitofp: case Intrinsic::experimental_constrained_sitofp: case Intrinsic::experimental_constrained_fptrunc: case Intrinsic::experimental_constrained_fpext: case Intrinsic::experimental_constrained_fcmp: case Intrinsic::experimental_constrained_fcmps: case Intrinsic::experimental_constrained_fmuladd: case Intrinsic::fmuladd: case Intrinsic::memset: case Intrinsic::memcpy: case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: case Intrinsic::dbg_declare: case Intrinsic::dbg_value: case Intrinsic::annotation: case Intrinsic::var_annotation: case Intrinsic::ptr_annotation: case Intrinsic::invariant_start: case Intrinsic::invariant_end: case Intrinsic::dbg_label: case Intrinsic::trap: case Intrinsic::arithmetic_fence: case Intrinsic::uadd_with_overflow: case Intrinsic::usub_with_overflow: return true; default: // Unknown intrinsics' declarations should always be translated return false; } } // Add decoration if needed SPIRVInstruction *addFPBuiltinDecoration(SPIRVModule *BM, IntrinsicInst *II, SPIRVInstruction *I) { const bool AllowFPMaxError = BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fp_max_error); assert(II->getCalledFunction()->getName().startswith("llvm.fpbuiltin")); // Add a new decoration for llvm.builtin intrinsics, if needed if (AllowFPMaxError) if (II->getAttributes().hasFnAttr("fpbuiltin-max-error")) { double F = 0.0; II->getAttributes() .getFnAttr("fpbuiltin-max-error") .getValueAsString() .getAsDouble(F); I->addDecorate(DecorationFPMaxErrorDecorationINTEL, convertFloatToSPIRVWord(F)); } return I; } // Performs mapping of LLVM IR rounding mode to SPIR-V rounding mode // Value *V is metadata argument of // llvm.experimental.constrained.* intrinsics SPIRVInstruction * LLVMToSPIRVBase::applyRoundingModeConstraint(Value *V, SPIRVInstruction *I) { StringRef RMode = cast(cast(V)->getMetadata())->getString(); if (RMode.endswith("tonearest")) I->addFPRoundingMode(FPRoundingModeRTE); else if (RMode.endswith("towardzero")) I->addFPRoundingMode(FPRoundingModeRTZ); else if (RMode.endswith("upward")) I->addFPRoundingMode(FPRoundingModeRTP); else if (RMode.endswith("downward")) I->addFPRoundingMode(FPRoundingModeRTN); return I; } static SPIRVWord getBuiltinIdForIntrinsic(Intrinsic::ID IID) { switch (IID) { // Note: In some cases the semantics of the OpenCL builtin are not identical // to the semantics of the corresponding LLVM IR intrinsic. The LLVM // intrinsics handled here assume the default floating point environment // (no unmasked exceptions, round-to-nearest-ties-even rounding mode) // and assume that the operations have no side effects (FP status flags // aren't maintained), so the OpenCL builtin behavior should be // acceptable. case Intrinsic::ceil: return OpenCLLIB::Ceil; case Intrinsic::copysign: return OpenCLLIB::Copysign; case Intrinsic::cos: return OpenCLLIB::Cos; case Intrinsic::exp: return OpenCLLIB::Exp; case Intrinsic::exp2: return OpenCLLIB::Exp2; case Intrinsic::fabs: return OpenCLLIB::Fabs; case Intrinsic::floor: return OpenCLLIB::Floor; case Intrinsic::fma: return OpenCLLIB::Fma; case Intrinsic::log: return OpenCLLIB::Log; case Intrinsic::log10: return OpenCLLIB::Log10; case Intrinsic::log2: return OpenCLLIB::Log2; case Intrinsic::maximum: return OpenCLLIB::Fmax; case Intrinsic::maxnum: return OpenCLLIB::Fmax; case Intrinsic::minimum: return OpenCLLIB::Fmin; case Intrinsic::minnum: return OpenCLLIB::Fmin; case Intrinsic::nearbyint: return OpenCLLIB::Rint; case Intrinsic::pow: return OpenCLLIB::Pow; case Intrinsic::powi: return OpenCLLIB::Pown; case Intrinsic::rint: return OpenCLLIB::Rint; case Intrinsic::round: return OpenCLLIB::Round; case Intrinsic::roundeven: return OpenCLLIB::Rint; case Intrinsic::sin: return OpenCLLIB::Sin; case Intrinsic::sqrt: return OpenCLLIB::Sqrt; case Intrinsic::trunc: return OpenCLLIB::Trunc; default: assert(false && "Builtin ID requested for Unhandled intrinsic!"); return 0; } } static SPIRVWord getNativeBuiltinIdForIntrinsic(Intrinsic::ID IID) { switch (IID) { case Intrinsic::cos: return OpenCLLIB::Native_cos; case Intrinsic::exp: return OpenCLLIB::Native_exp; case Intrinsic::exp2: return OpenCLLIB::Native_exp2; case Intrinsic::log: return OpenCLLIB::Native_log; case Intrinsic::log10: return OpenCLLIB::Native_log10; case Intrinsic::log2: return OpenCLLIB::Native_log2; case Intrinsic::sin: return OpenCLLIB::Native_sin; case Intrinsic::sqrt: return OpenCLLIB::Native_sqrt; default: return getBuiltinIdForIntrinsic(IID); } } static bool allowsApproxFunction(IntrinsicInst *II) { auto *Ty = II->getType(); // OpenCL native_* built-ins only support single precision data type return II->hasApproxFunc() && (Ty->isFloatTy() || (Ty->isVectorTy() && cast(Ty)->getElementType()->isFloatTy())); } SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II, SPIRVBasicBlock *BB) { auto GetMemoryAccess = [](MemIntrinsic *MI, bool AllowTwoMemAccessMasks) -> std::vector { std::vector MemoryAccess(1, MemoryAccessMaskNone); if (SPIRVWord AlignVal = MI->getDestAlignment()) { MemoryAccess[0] |= MemoryAccessAlignedMask; if (auto *MTI = dyn_cast(MI)) { SPIRVWord SourceAlignVal = MTI->getSourceAlignment(); assert(SourceAlignVal && "Missed Source alignment!"); // In a case when alignment of source differs from dest one // we either preserve both (allowed since SPIR-V 1.4), or the least // value is guaranteed anyway. if (AllowTwoMemAccessMasks) { if (AlignVal != SourceAlignVal) { MemoryAccess.push_back(AlignVal); MemoryAccess.push_back(MemoryAccessAlignedMask); AlignVal = SourceAlignVal; } } else { AlignVal = std::min(AlignVal, SourceAlignVal); } } MemoryAccess.push_back(AlignVal); } if (MI->isVolatile()) MemoryAccess[0] |= MemoryAccessVolatileMask; return MemoryAccess; }; // LLVM intrinsics with known translation to SPIR-V are handled here. They // also must be registered at isKnownIntrinsic function in order to make // -spirv-allow-unknown-intrinsics work correctly. auto IID = II->getIntrinsicID(); switch (IID) { case Intrinsic::fabs: case Intrinsic::fma: case Intrinsic::maxnum: case Intrinsic::minnum: case Intrinsic::fmuladd: { Type *Ty = II->getType(); if (Ty->isBFloatTy()) BM->addCapability(internal::CapabilityBFloat16ArithmeticINTEL); break; } default: break; } switch (IID) { case Intrinsic::assume: { // llvm.assume translation is currently supported only within // SPV_KHR_expect_assume extension, ignore it otherwise, since it's // an optimization hint if (BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_expect_assume)) { SPIRVValue *Condition = transValue(II->getArgOperand(0), BB); return BM->addAssumeTrueKHRInst(Condition, BB); } return nullptr; } case Intrinsic::bitreverse: { BM->addCapability(CapabilityShader); SPIRVType *Ty = transType(II->getType()); SPIRVValue *Op = transValue(II->getArgOperand(0), BB); return BM->addUnaryInst(OpBitReverse, Ty, Op, BB); } // Unary FP intrinsic case Intrinsic::ceil: case Intrinsic::cos: case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::fabs: case Intrinsic::floor: case Intrinsic::log: case Intrinsic::log10: case Intrinsic::log2: case Intrinsic::nearbyint: case Intrinsic::rint: case Intrinsic::round: case Intrinsic::roundeven: case Intrinsic::sin: case Intrinsic::sqrt: case Intrinsic::trunc: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; const SPIRVWord ExtOp = allowsApproxFunction(II) ? getNativeBuiltinIdForIntrinsic(IID) : getBuiltinIdForIntrinsic(IID); SPIRVType *STy = transType(II->getType()); std::vector Ops(1, transValue(II->getArgOperand(0), BB)); return BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } // Binary FP intrinsics case Intrinsic::copysign: case Intrinsic::pow: case Intrinsic::powi: case Intrinsic::maximum: case Intrinsic::maxnum: case Intrinsic::minimum: case Intrinsic::minnum: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; const SPIRVWord ExtOp = allowsApproxFunction(II) ? getNativeBuiltinIdForIntrinsic(IID) : getBuiltinIdForIntrinsic(IID); SPIRVType *STy = transType(II->getType()); std::vector Ops{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB)}; return BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } case Intrinsic::umin: case Intrinsic::umax: case Intrinsic::smin: case Intrinsic::smax: { Type *BoolTy = IntegerType::getInt1Ty(M->getContext()); SPIRVValue *FirstArgVal = transValue(II->getArgOperand(0), BB); SPIRVValue *SecondArgVal = transValue(II->getArgOperand(1), BB); const Op OC = (IID == Intrinsic::smin) ? OpSLessThan : ((IID == Intrinsic::smax) ? OpSGreaterThan : ((IID == Intrinsic::umin) ? OpULessThan : OpUGreaterThan)); if (auto *VecTy = dyn_cast(II->getArgOperand(0)->getType())) BoolTy = VectorType::get(BoolTy, VecTy->getElementCount()); SPIRVValue *Cmp = BM->addCmpInst(OC, transType(BoolTy), FirstArgVal, SecondArgVal, BB); return BM->addSelectInst(Cmp, FirstArgVal, SecondArgVal, BB); } case Intrinsic::fma: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; SPIRVWord ExtOp = OpenCLLIB::Fma; SPIRVType *STy = transType(II->getType()); std::vector Ops{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), transValue(II->getArgOperand(2), BB)}; return BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } case Intrinsic::abs: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; // LLVM has only one version of abs and it is only for signed integers. We // unconditionally choose SAbs here SPIRVWord ExtOp = OpenCLLIB::SAbs; SPIRVType *STy = transType(II->getType()); std::vector Ops(1, transValue(II->getArgOperand(0), BB)); return BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } case Intrinsic::ctpop: { return BM->addUnaryInst(OpBitCount, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); } case Intrinsic::ctlz: case Intrinsic::cttz: { const SPIRVWord ExtOp = IID == Intrinsic::ctlz ? OpenCLLIB::Clz : OpenCLLIB::Ctz; SPIRVType *Ty = transType(II->getType()); std::vector Ops(1, transValue(II->getArgOperand(0), BB)); return BM->addExtInst(Ty, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } case Intrinsic::expect: { // llvm.expect translation is currently supported only within // SPV_KHR_expect_assume extension, replace it with a translated value of #0 // operand otherwise, since it's an optimization hint SPIRVValue *Value = transValue(II->getArgOperand(0), BB); if (BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_expect_assume)) { SPIRVType *Ty = transType(II->getType()); SPIRVValue *ExpectedValue = transValue(II->getArgOperand(1), BB); return BM->addExpectKHRInst(Ty, Value, ExpectedValue, BB); } return Value; } case Intrinsic::experimental_constrained_fadd: { auto BI = BM->addBinaryInst(OpFAdd, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return applyRoundingModeConstraint(II->getOperand(2), BI); } case Intrinsic::experimental_constrained_fsub: { auto BI = BM->addBinaryInst(OpFSub, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return applyRoundingModeConstraint(II->getOperand(2), BI); } case Intrinsic::experimental_constrained_fmul: { auto BI = BM->addBinaryInst(OpFMul, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return applyRoundingModeConstraint(II->getOperand(2), BI); } case Intrinsic::experimental_constrained_fdiv: { auto BI = BM->addBinaryInst(OpFDiv, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return applyRoundingModeConstraint(II->getOperand(2), BI); } case Intrinsic::experimental_constrained_frem: { auto BI = BM->addBinaryInst(OpFRem, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return applyRoundingModeConstraint(II->getOperand(2), BI); } case Intrinsic::experimental_constrained_fma: { std::vector Args{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), transValue(II->getArgOperand(2), BB)}; auto BI = BM->addExtInst(transType(II->getType()), BM->getExtInstSetId(SPIRVEIS_OpenCL), OpenCLLIB::Fma, Args, BB); return applyRoundingModeConstraint(II->getOperand(3), BI); } case Intrinsic::experimental_constrained_fptoui: { return BM->addUnaryInst(OpConvertFToU, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); } case Intrinsic::experimental_constrained_fptosi: { return BM->addUnaryInst(OpConvertFToS, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); } case Intrinsic::experimental_constrained_uitofp: { auto BI = BM->addUnaryInst(OpConvertUToF, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); return applyRoundingModeConstraint(II->getOperand(1), BI); } case Intrinsic::experimental_constrained_sitofp: { auto BI = BM->addUnaryInst(OpConvertSToF, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); return applyRoundingModeConstraint(II->getOperand(1), BI); } case Intrinsic::experimental_constrained_fpext: { return BM->addUnaryInst(OpFConvert, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); } case Intrinsic::experimental_constrained_fptrunc: { auto BI = BM->addUnaryInst(OpFConvert, transType(II->getType()), transValue(II->getArgOperand(0), BB), BB); return applyRoundingModeConstraint(II->getOperand(1), BI); } case Intrinsic::experimental_constrained_fcmp: case Intrinsic::experimental_constrained_fcmps: { auto MetaMod = cast(II->getOperand(2))->getMetadata(); Op CmpTypeOp = StringSwitch(cast(MetaMod)->getString()) .Case("oeq", OpFOrdEqual) .Case("ogt", OpFOrdGreaterThan) .Case("oge", OpFOrdGreaterThanEqual) .Case("olt", OpFOrdLessThan) .Case("ole", OpFOrdLessThanEqual) .Case("one", OpFOrdNotEqual) .Case("ord", OpOrdered) .Case("ueq", OpFUnordEqual) .Case("ugt", OpFUnordGreaterThan) .Case("uge", OpFUnordGreaterThanEqual) .Case("ult", OpFUnordLessThan) .Case("ule", OpFUnordLessThanEqual) .Case("une", OpFUnordNotEqual) .Case("uno", OpUnordered) .Default(OpNop); assert(CmpTypeOp != OpNop && "Invalid condition code!"); return BM->addCmpInst(CmpTypeOp, transType(II->getType()), transValue(II->getOperand(0), BB), transValue(II->getOperand(1), BB), BB); } case Intrinsic::experimental_constrained_fmuladd: { SPIRVType *Ty = transType(II->getType()); SPIRVValue *Mul = BM->addBinaryInst(OpFMul, Ty, transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); auto BI = BM->addBinaryInst(OpFAdd, Ty, Mul, transValue(II->getArgOperand(2), BB), BB); return applyRoundingModeConstraint(II->getOperand(3), BI); } case Intrinsic::fmuladd: { // For llvm.fmuladd.* fusion is not guaranteed. If a fused multiply-add // is required the corresponding llvm.fma.* intrinsic function should be // used instead. // If allowed, let's replace llvm.fmuladd.* with mad from OpenCL extended // instruction set, as it has the same semantic for FULL_PROFILE OpenCL // devices (implementation-defined for EMBEDDED_PROFILE). if (BM->shouldReplaceLLVMFmulAddWithOpenCLMad()) { std::vector Ops{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), transValue(II->getArgOperand(2), BB)}; return BM->addExtInst(transType(II->getType()), BM->getExtInstSetId(SPIRVEIS_OpenCL), OpenCLLIB::Mad, Ops, BB); } // Otherwise, just break llvm.fmuladd.* into a pair of fmul + fadd SPIRVType *Ty = transType(II->getType()); SPIRVValue *Mul = BM->addBinaryInst(OpFMul, Ty, transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return BM->addBinaryInst(OpFAdd, Ty, Mul, transValue(II->getArgOperand(2), BB), BB); } case Intrinsic::usub_sat: { // usub.sat(a, b) -> (a > b) ? a - b : 0 SPIRVType *Ty = transType(II->getType()); Type *BoolTy = IntegerType::getInt1Ty(M->getContext()); SPIRVValue *FirstArgVal = transValue(II->getArgOperand(0), BB); SPIRVValue *SecondArgVal = transValue(II->getArgOperand(1), BB); SPIRVValue *Sub = BM->addBinaryInst(OpISub, Ty, FirstArgVal, SecondArgVal, BB); SPIRVValue *Cmp = BM->addCmpInst(OpUGreaterThan, transType(BoolTy), FirstArgVal, SecondArgVal, BB); SPIRVValue *Zero = transValue(Constant::getNullValue(II->getType()), BB); return BM->addSelectInst(Cmp, Sub, Zero, BB); } case Intrinsic::uadd_with_overflow: { return BM->addBinaryInst(OpIAddCarry, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); } case Intrinsic::usub_with_overflow: { return BM->addBinaryInst(OpISubBorrow, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); } case Intrinsic::memset: { // Generally there is no direct mapping of memset to SPIR-V. But it turns // out that memset is emitted by Clang for initialization in default // constructors so we need some basic support. The code below only handles // cases with constant value and constant length. MemSetInst *MSI = cast(II); Value *Val = MSI->getValue(); if (!isa(Val)) { assert(false && "Can't translate llvm.memset with non-const `value` argument"); return nullptr; } Value *Len = MSI->getLength(); if (!isa(Len)) { assert(false && "Can't translate llvm.memset with non-const `length` argument"); return nullptr; } uint64_t NumElements = static_cast(Len)->getZExtValue(); auto *AT = ArrayType::get(Val->getType(), NumElements); SPIRVTypeArray *CompositeTy = static_cast(transType(AT)); SPIRVValue *Init; if (cast(Val)->isZeroValue()) { Init = BM->addNullConstant(CompositeTy); } else { // On 32-bit systems, size_type of std::vector is not a 64-bit type. Let's // assume that we won't encounter memset for more than 2^32 elements and // insert explicit cast to avoid possible warning/error about narrowing // conversion auto TNumElts = static_cast::size_type>(NumElements); std::vector Elts(TNumElts, transValue(Val, BB)); Init = BM->addCompositeConstant(CompositeTy, Elts); } SPIRVType *VarTy = transPointerType(AT, SPIRV::SPIRAS_Constant); SPIRVValue *Var = BM->addVariable(VarTy, /*isConstant*/ true, spv::internal::LinkageTypeInternal, Init, "", StorageClassUniformConstant, nullptr); SPIRVType *SourceTy = transPointerType(Val->getType(), SPIRV::SPIRAS_Constant); SPIRVValue *Source = BM->addUnaryInst(OpBitcast, SourceTy, Var, BB); SPIRVValue *Target = transValue(MSI->getRawDest(), BB); return BM->addCopyMemorySizedInst( Target, Source, CompositeTy->getLength(), GetMemoryAccess(MSI, BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)), BB); } break; case Intrinsic::memcpy: return BM->addCopyMemorySizedInst( transValue(II->getOperand(0), BB), transValue(II->getOperand(1), BB), transValue(II->getOperand(2), BB), GetMemoryAccess(cast(II), BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)), BB); case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: { Op OC = (II->getIntrinsicID() == Intrinsic::lifetime_start) ? OpLifetimeStart : OpLifetimeStop; int64_t Size = dyn_cast(II->getOperand(0))->getSExtValue(); if (Size == -1) Size = 0; return BM->addLifetimeInst(OC, transValue(II->getOperand(1), BB), Size, BB); } // We don't want to mix translation of regular code and debug info, because // it creates a mess, therefore translation of debug intrinsics is // postponed until LLVMToSPIRVDbgTran::finalizeDebug...() methods. case Intrinsic::dbg_declare: return DbgTran->createDebugDeclarePlaceholder(cast(II), BB); case Intrinsic::dbg_value: return DbgTran->createDebugValuePlaceholder(cast(II), BB); case Intrinsic::annotation: { SPIRVType *Ty = transType(II->getType()); GetElementPtrInst *GEP = dyn_cast(II->getArgOperand(1)); if (!GEP) return nullptr; Constant *C = cast(GEP->getOperand(0)); StringRef AnnotationString; getConstantStringInfo(C, AnnotationString); if (AnnotationString == kOCLBuiltinName::FPGARegIntel) { if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_reg)) return BM->addFPGARegINTELInst(Ty, transValue(II->getOperand(0), BB), BB); else return transValue(II->getOperand(0), BB); } return nullptr; } case Intrinsic::var_annotation: { SPIRVValue *SV; if (auto *BI = dyn_cast(II->getArgOperand(0))) { SV = transValue(BI->getOperand(0), BB); } else { SV = transValue(II->getOperand(0), BB); } std::string AnnotationString; processAnnotationString(II, AnnotationString); DecorationsInfoVec Decorations = tryParseAnnotationString(BM, AnnotationString).MemoryAttributesVec; // If we didn't find any IntelFPGA-specific decorations, let's add the whole // annotation string as UserSemantic Decoration if (Decorations.empty()) { SV->addDecorate( new SPIRVDecorateUserSemanticAttr(SV, AnnotationString.c_str())); } else { addAnnotationDecorations(SV, Decorations); } return SV; } // The layout of llvm.ptr.annotation is: // declare iN* @llvm.ptr.annotation.p
iN( // iN* , i8* , i8* , i32 , i8* ) // where N is a power of two number, // first i8* stands for the annotation itself, // second i8* is for the location (file name), // i8* is a pointer on a GV, which can carry optinal variadic // clang::annotation attribute expression arguments. case Intrinsic::ptr_annotation: { // Strip all bitcast and addrspace casts from the pointer argument: // llvm annotation intrinsic only takes i8*, so the original pointer // probably had to loose its addrspace and its original type. Value *AnnotSubj = II->getArgOperand(0); while (isa(AnnotSubj) || isa(AnnotSubj)) { AnnotSubj = cast(AnnotSubj)->getOperand(0); } std::string AnnotationString; processAnnotationString(II, AnnotationString); AnnotationDecorations Decorations = tryParseAnnotationString(BM, AnnotationString); // If the pointer is a GEP on a struct, then we have to emit a member // decoration for the GEP-accessed struct, or a memory access decoration // for the GEP itself. auto *GI = dyn_cast(AnnotSubj); if (GI && isa(GI->getSourceElementType())) { auto *Ty = transType(GI->getSourceElementType()); auto *ResPtr = transValue(GI, BB); SPIRVWord MemberNumber = dyn_cast(GI->getOperand(2))->getZExtValue(); // If we didn't find any IntelFPGA-specific decorations, let's add the // whole annotation string as UserSemantic Decoration if (Decorations.MemoryAttributesVec.empty() && Decorations.MemoryAccessesVec.empty()) { // TODO: Is there a way to detect that the annotation belongs solely // to struct member memory atributes or struct member memory access // controls? This would allow emitting just the necessary decoration. Ty->addMemberDecorate(new SPIRVMemberDecorateUserSemanticAttr( Ty, MemberNumber, AnnotationString.c_str())); ResPtr->addDecorate(new SPIRVDecorateUserSemanticAttr( ResPtr, AnnotationString.c_str())); } else { addAnnotationDecorationsForStructMember( Ty, MemberNumber, Decorations.MemoryAttributesVec); // Apply the LSU parameter decoration to the pointer result of a GEP // to the given struct member (InBoundsPtrAccessChain in SPIR-V). // Decorating the member itself with a MemberDecoration is not feasible, // because multiple accesses to the struct-held memory can require // different LSU parameters. addAnnotationDecorations(ResPtr, Decorations.MemoryAccessesVec); } II->replaceAllUsesWith(II->getOperand(0)); } else { // TODO: Check for opaque pointer requirements. auto *Ty = transType(II->getType()); auto *BI = dyn_cast(II->getOperand(0)); if (AnnotationString == kOCLBuiltinName::FPGARegIntel) { if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_reg)) return BM->addFPGARegINTELInst(Ty, transValue(BI, BB), BB); return transValue(BI, BB); } else { // Memory accesses to a standalone pointer variable auto *DecSubj = transValue(II->getArgOperand(0), BB); if (Decorations.MemoryAccessesVec.empty()) DecSubj->addDecorate(new SPIRVDecorateUserSemanticAttr( DecSubj, AnnotationString.c_str())); else // Apply the LSU parameter decoration to the pointer result of an // instruction. Note it's the address to the accessed memory that's // loaded from the original pointer variable, and not the value // accessed by the latter. addAnnotationDecorations(DecSubj, Decorations.MemoryAccessesVec); II->replaceAllUsesWith(II->getOperand(0)); } } return nullptr; } case Intrinsic::stacksave: { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_variable_length_array)) { auto *Ty = transPointerType(Type::getInt8Ty(M->getContext()), 0); return BM->addInstTemplate(OpSaveMemoryINTEL, BB, Ty); } BM->getErrorLog().checkError( BM->isUnknownIntrinsicAllowed(II), SPIRVEC_InvalidFunctionCall, II, "Translation of llvm.stacksave intrinsic requires " "SPV_INTEL_variable_length_array extension or " "-spirv-allow-unknown-intrinsics option."); break; } case Intrinsic::stackrestore: { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_variable_length_array)) { auto *Ptr = transValue(II->getArgOperand(0), BB); return BM->addInstTemplate(OpRestoreMemoryINTEL, {Ptr->getId()}, BB, nullptr); } BM->getErrorLog().checkError( BM->isUnknownIntrinsicAllowed(II), SPIRVEC_InvalidFunctionCall, II, "Translation of llvm.restore intrinsic requires " "SPV_INTEL_variable_length_array extension or " "-spirv-allow-unknown-intrinsics option."); break; } // We can just ignore/drop some intrinsics, like optimizations hint. case Intrinsic::experimental_noalias_scope_decl: case Intrinsic::invariant_start: case Intrinsic::invariant_end: case Intrinsic::dbg_label: // llvm.trap intrinsic is not implemented. But for now don't crash. This // change is pending the trap/abort intrinsic implementation. case Intrinsic::trap: // llvm.instrprof.* intrinsics are not supported case Intrinsic::instrprof_increment: case Intrinsic::instrprof_increment_step: case Intrinsic::instrprof_value_profile: return nullptr; case Intrinsic::is_constant: { auto *CO = dyn_cast(II->getOperand(0)); if (CO && isManifestConstant(CO)) return transValue(ConstantInt::getTrue(II->getType()), BB, false); else return transValue(ConstantInt::getFalse(II->getType()), BB, false); } case Intrinsic::arithmetic_fence: { SPIRVType *Ty = transType(II->getType()); SPIRVValue *Op = transValue(II->getArgOperand(0), BB); if (BM->isAllowedToUseExtension(ExtensionID::SPV_EXT_arithmetic_fence)) { BM->addCapability(CapabilityArithmeticFenceEXT); BM->addExtension(ExtensionID::SPV_EXT_arithmetic_fence); return BM->addUnaryInst(OpArithmeticFenceEXT, Ty, Op, BB); } if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_arithmetic_fence)) { // Note: SPV_INTEL_arithmetic_fence was unpublished and superseded by // SPV_EXT_arithmetic_fence. BM->addCapability(CapabilityArithmeticFenceEXT); BM->addExtension(ExtensionID::SPV_INTEL_arithmetic_fence); return BM->addUnaryInst(OpArithmeticFenceEXT, Ty, Op, BB); } return Op; } case Intrinsic::masked_gather: { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_masked_gather_scatter)) { BM->getErrorLog().checkError( BM->isUnknownIntrinsicAllowed(II), SPIRVEC_InvalidFunctionCall, II, "Translation of llvm.masked.gather intrinsic requires " "SPV_INTEL_masked_gather_scatter extension or " "-spirv-allow-unknown-intrinsics option."); return nullptr; } SPIRVType *Ty = transType(II->getType()); auto *PtrVector = transValue(II->getArgOperand(0), BB); uint32_t Alignment = cast(II->getArgOperand(1))->getZExtValue(); auto *Mask = transValue(II->getArgOperand(2), BB); auto *FillEmpty = transValue(II->getArgOperand(3), BB); std::vector Ops = {PtrVector->getId(), Alignment, Mask->getId(), FillEmpty->getId()}; return BM->addInstTemplate(internal::OpMaskedGatherINTEL, Ops, BB, Ty); } case Intrinsic::masked_scatter: { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_masked_gather_scatter)) { BM->getErrorLog().checkError( BM->isUnknownIntrinsicAllowed(II), SPIRVEC_InvalidFunctionCall, II, "Translation of llvm.masked.scatter intrinsic requires " "SPV_INTEL_masked_gather_scatter extension or " "-spirv-allow-unknown-intrinsics option."); return nullptr; } auto *InputVector = transValue(II->getArgOperand(0), BB); auto *PtrVector = transValue(II->getArgOperand(1), BB); uint32_t Alignment = cast(II->getArgOperand(2))->getZExtValue(); auto *Mask = transValue(II->getArgOperand(3), BB); std::vector Ops = {InputVector->getId(), PtrVector->getId(), Alignment, Mask->getId()}; return BM->addInstTemplate(internal::OpMaskedScatterINTEL, Ops, BB, nullptr); } default: if (auto *BVar = transFPBuiltinIntrinsicInst(II, BB)) return BVar; if (BM->isUnknownIntrinsicAllowed(II)) return BM->addCallInst( transFunctionDecl(II->getCalledFunction()), transArguments(II, BB, SPIRVEntry::createUnique(OpFunctionCall).get()), BB); else // Other LLVM intrinsics shouldn't get to SPIRV, because they // can't be represented in SPIRV or aren't implemented yet. BM->SPIRVCK(false, InvalidFunctionCall, II->getCalledOperand()->getName().str()); } return nullptr; } LLVMToSPIRVBase::FPBuiltinType LLVMToSPIRVBase::getFPBuiltinType(IntrinsicInst *II, StringRef &OpName) { StringRef Name = II->getCalledFunction()->getName(); if (!Name.startswith("llvm.fpbuiltin")) return FPBuiltinType::UNKNOWN; Name.consume_front("llvm.fpbuiltin."); OpName = Name.split('.').first; FPBuiltinType Type = StringSwitch(OpName) .Cases("fadd", "fsub", "fmul", "fdiv", "frem", FPBuiltinType::REGULAR_MATH) .Cases("sin", "cos", "tan", FPBuiltinType::EXT_1OPS) .Cases("sinh", "cosh", "tanh", FPBuiltinType::EXT_1OPS) .Cases("asin", "acos", "atan", FPBuiltinType::EXT_1OPS) .Cases("asinh", "acosh", "atanh", FPBuiltinType::EXT_1OPS) .Cases("exp", "exp2", "exp10", "expm1", FPBuiltinType::EXT_1OPS) .Cases("log", "log2", "log10", "log1p", FPBuiltinType::EXT_1OPS) .Cases("sqrt", "rsqrt", "erf", "erfc", FPBuiltinType::EXT_1OPS) .Cases("atan2", "pow", "hypot", "ldexp", FPBuiltinType::EXT_2OPS) .Case("sincos", FPBuiltinType::EXT_3OPS) .Default(FPBuiltinType::UNKNOWN); return Type; } SPIRVValue *LLVMToSPIRVBase::transFPBuiltinIntrinsicInst(IntrinsicInst *II, SPIRVBasicBlock *BB) { StringRef OpName; auto FPBuiltinTypeVal = getFPBuiltinType(II, OpName); if (FPBuiltinTypeVal == FPBuiltinType::UNKNOWN) return nullptr; switch (FPBuiltinTypeVal) { case FPBuiltinType::REGULAR_MATH: { auto BinOp = StringSwitch(OpName) .Case("fadd", OpFAdd) .Case("fsub", OpFSub) .Case("fmul", OpFMul) .Case("fdiv", OpFDiv) .Case("frem", OpFRem) .Default(OpUndef); auto *BI = BM->addBinaryInst(BinOp, transType(II->getType()), transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), BB); return addFPBuiltinDecoration(BM, II, BI); } case FPBuiltinType::EXT_1OPS: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; SPIRVType *STy = transType(II->getType()); std::vector Ops(1, transValue(II->getArgOperand(0), BB)); auto ExtOp = StringSwitch(OpName) .Case("sin", OpenCLLIB::Sin) .Case("cos", OpenCLLIB::Cos) .Case("tan", OpenCLLIB::Tan) .Case("sinh", OpenCLLIB::Sinh) .Case("cosh", OpenCLLIB::Cosh) .Case("tanh", OpenCLLIB::Tanh) .Case("asin", OpenCLLIB::Asin) .Case("acos", OpenCLLIB::Acos) .Case("atan", OpenCLLIB::Atan) .Case("asinh", OpenCLLIB::Asinh) .Case("acosh", OpenCLLIB::Acosh) .Case("atanh", OpenCLLIB::Atanh) .Case("exp", OpenCLLIB::Exp) .Case("exp2", OpenCLLIB::Exp2) .Case("exp10", OpenCLLIB::Exp10) .Case("expm1", OpenCLLIB::Expm1) .Case("log", OpenCLLIB::Log) .Case("log2", OpenCLLIB::Log2) .Case("log10", OpenCLLIB::Log10) .Case("log1p", OpenCLLIB::Log1p) .Case("sqrt", OpenCLLIB::Sqrt) .Case("rsqrt", OpenCLLIB::Rsqrt) .Case("erf", OpenCLLIB::Erf) .Case("erfc", OpenCLLIB::Erfc) .Default(SPIRVWORD_MAX); assert(ExtOp != SPIRVWORD_MAX); auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); return addFPBuiltinDecoration(BM, II, BI); } case FPBuiltinType::EXT_2OPS: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; SPIRVType *STy = transType(II->getType()); std::vector Ops{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB)}; auto ExtOp = StringSwitch(OpName) .Case("atan2", OpenCLLIB::Atan2) .Case("hypot", OpenCLLIB::Hypot) .Case("pow", OpenCLLIB::Pow) .Case("ldexp", OpenCLLIB::Ldexp) .Default(SPIRVWORD_MAX); assert(ExtOp != SPIRVWORD_MAX); auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); return addFPBuiltinDecoration(BM, II, BI); } case FPBuiltinType::EXT_3OPS: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; SPIRVType *STy = transType(II->getType()); std::vector Ops{transValue(II->getArgOperand(0), BB), transValue(II->getArgOperand(1), BB), transValue(II->getArgOperand(2), BB)}; auto ExtOp = StringSwitch(OpName) .Case("sincos", OpenCLLIB::Sincos) .Default(SPIRVWORD_MAX); assert(ExtOp != SPIRVWORD_MAX); auto *BI = BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); return addFPBuiltinDecoration(BM, II, BI); } default: return nullptr; } return nullptr; } SPIRVValue *LLVMToSPIRVBase::transFenceInst(FenceInst *FI, SPIRVBasicBlock *BB) { SPIRVWord MemorySemantics; // Fence ordering may only be Acquire, Release, AcquireRelease, or // SequentiallyConsistent switch (FI->getOrdering()) { case llvm::AtomicOrdering::Acquire: MemorySemantics = MemorySemanticsAcquireMask; break; case llvm::AtomicOrdering::Release: MemorySemantics = MemorySemanticsReleaseMask; break; case llvm::AtomicOrdering::AcquireRelease: MemorySemantics = MemorySemanticsAcquireReleaseMask; break; case llvm::AtomicOrdering::SequentiallyConsistent: MemorySemantics = MemorySemanticsSequentiallyConsistentMask; break; default: assert(false && "Unexpected fence ordering"); MemorySemantics = SPIRVWORD_MAX; break; } Module *M = FI->getParent()->getModule(); // Treat all llvm.fence instructions as having CrossDevice scope: SPIRVValue *RetScope = transConstant(getUInt32(M, ScopeCrossDevice)); SPIRVValue *Val = transConstant(getUInt32(M, MemorySemantics)); assert(RetScope && Val && "RetScope and Value are not constants"); return BM->addMemoryBarrierInst(static_cast(RetScope->getId()), Val->getId(), BB); } SPIRVValue *LLVMToSPIRVBase::transCallInst(CallInst *CI, SPIRVBasicBlock *BB) { assert(CI); Function *F = CI->getFunction(); if (isa(CI->getCalledOperand()) && BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_inline_assembly)) { // Inline asm is opaque, so we cannot reason about its FP contraction // requirements. SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName() << ": inline asm " << *CI << '\n'); joinFPContract(F, FPContract::DISABLED); return transAsmCallINTEL(CI, BB); } if (CI->isIndirectCall()) { // The function is not known in advance SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName() << ": indirect call " << *CI << '\n'); joinFPContract(F, FPContract::DISABLED); return transIndirectCallInst(CI, BB); } return transDirectCallInst(CI, BB); } SPIRVValue *LLVMToSPIRVBase::transDirectCallInst(CallInst *CI, SPIRVBasicBlock *BB) { SPIRVExtInstSetKind ExtSetKind = SPIRVEIS_Count; SPIRVWord ExtOp = SPIRVWORD_MAX; llvm::Function *F = CI->getCalledFunction(); auto MangledName = F->getName(); StringRef DemangledName; if (MangledName.startswith(SPCV_CAST) || MangledName == SAMPLER_INIT) return oclTransSpvcCastSampler(CI, BB); if (oclIsBuiltin(MangledName, DemangledName) || isDecoratedSPIRVFunc(F, DemangledName)) { if (auto BV = transBuiltinToConstant(DemangledName, CI)) return BV; if (auto BV = transBuiltinToInst(DemangledName, CI, BB)) return BV; } SmallVector Dec; if (isBuiltinTransToExtInst(CI->getCalledFunction(), &ExtSetKind, &ExtOp, &Dec)) { if (const auto *FirstArg = F->getArg(0)) { const auto *Type = FirstArg->getType(); if (Type->isBFloatTy()) BM->addCapability(internal::CapabilityBFloat16ArithmeticINTEL); } if (DemangledName.find("__spirv_ocl_printf") != StringRef::npos) { auto *FormatStrPtr = cast(CI->getArgOperand(0)->getType()); if (FormatStrPtr->getAddressSpace() != SPIR::TypeAttributeEnum::ATTR_CONST) { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_EXT_relaxed_printf_string_address_space)) { std::string ErrorStr = "Either SPV_EXT_relaxed_printf_string_address_space extension " "should be allowed to translate this module, because this LLVM " "module contains the printf function with format string, whose " "address space is not equal to 2 (constant)."; getErrorLog().checkError(false, SPIRVEC_RequiresExtension, CI, ErrorStr); } BM->addExtension( ExtensionID::SPV_EXT_relaxed_printf_string_address_space); } } // TODO: Check for opaque pointer requirements. return addDecorations( BM->addExtInst( transType(CI->getType()), BM->getExtInstSetId(ExtSetKind), ExtOp, transArguments(CI, BB, SPIRVEntry::createUnique(ExtSetKind, ExtOp).get()), BB), Dec); } Function *Callee = CI->getCalledFunction(); if (Callee->isDeclaration()) { SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName().str() << ": call to an undefined function " << *CI << '\n'); joinFPContract(CI->getFunction(), FPContract::DISABLED); } else { FPContract CalleeFPC = getFPContract(Callee); joinFPContract(CI->getFunction(), CalleeFPC); if (CalleeFPC == FPContract::DISABLED) { SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName().str() << ": call to a function with disabled contraction: " << *CI << '\n'); } } return BM->addCallInst( transFunctionDecl(Callee), transArguments(CI, BB, SPIRVEntry::createUnique(OpFunctionCall).get()), BB); } SPIRVValue *LLVMToSPIRVBase::transIndirectCallInst(CallInst *CI, SPIRVBasicBlock *BB) { if (BM->getErrorLog().checkError( BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_function_pointers), SPIRVEC_FunctionPointers, CI)) { // TODO: Check for opaque pointer requirements. return BM->addIndirectCallInst( transValue(CI->getCalledOperand(), BB), transType(CI->getType()), transArguments(CI, BB, SPIRVEntry::createUnique(OpFunctionCall).get()), BB); } return nullptr; } SPIRVValue *LLVMToSPIRVBase::transAsmINTEL(InlineAsm *IA) { assert(IA); // TODO: intention here is to provide information about actual target // but in fact spir-64 is substituted as triple when translator works // eventually we need to fix it (not urgent) StringRef TripleStr(M->getTargetTriple()); auto AsmTarget = static_cast( BM->getOrAddAsmTargetINTEL(TripleStr.str())); auto SIA = BM->addAsmINTEL( static_cast(transType(IA->getFunctionType())), AsmTarget, IA->getAsmString(), IA->getConstraintString()); if (IA->hasSideEffects()) SIA->addDecorate(DecorationSideEffectsINTEL); return SIA; } SPIRVValue *LLVMToSPIRVBase::transAsmCallINTEL(CallInst *CI, SPIRVBasicBlock *BB) { assert(CI); auto IA = cast(CI->getCalledOperand()); return BM->addAsmCallINTELInst( static_cast(transValue(IA, BB, false)), transArguments(CI, BB, SPIRVEntry::createUnique(OpAsmCallINTEL).get()), BB); } bool LLVMToSPIRVBase::transAddressingMode() { Triple TargetTriple(M->getTargetTriple()); if (TargetTriple.isArch32Bit()) BM->setAddressingModel(AddressingModelPhysical32); else BM->setAddressingModel(AddressingModelPhysical64); // Physical addressing model requires Addresses capability BM->addCapability(CapabilityAddresses); return true; } std::vector LLVMToSPIRVBase::transValue(const std::vector &Args, SPIRVBasicBlock *BB) { std::vector BArgs; for (auto &I : Args) BArgs.push_back(transValue(I, BB)); return BArgs; } std::vector LLVMToSPIRVBase::transValue(const std::vector &Args, SPIRVBasicBlock *BB, SPIRVEntry *Entry) { std::vector Operands; for (size_t I = 0, E = Args.size(); I != E; ++I) { Operands.push_back(Entry->isOperandLiteral(I) ? cast(Args[I])->getZExtValue() : transValue(Args[I], BB)->getId()); } return Operands; } std::vector LLVMToSPIRVBase::transArguments(CallInst *CI, SPIRVBasicBlock *BB, SPIRVEntry *Entry) { return transValue(getArguments(CI), BB, Entry); } SPIRVWord LLVMToSPIRVBase::transFunctionControlMask(Function *F) { SPIRVWord FCM = 0; SPIRSPIRVFuncCtlMaskMap::foreach ( [&](Attribute::AttrKind Attr, SPIRVFunctionControlMaskKind Mask) { if (F->hasFnAttribute(Attr)) { if (Attr == Attribute::OptimizeNone) { if (BM->isAllowedToUseExtension(ExtensionID::SPV_EXT_optnone)) { BM->addExtension(ExtensionID::SPV_EXT_optnone); BM->addCapability(CapabilityOptNoneEXT); } else if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_optnone)) { BM->addExtension(ExtensionID::SPV_INTEL_optnone); BM->addCapability(CapabilityOptNoneINTEL); } else return; } FCM |= Mask; } }); return FCM; } void LLVMToSPIRVBase::transGlobalAnnotation(GlobalVariable *V) { SPIRVDBG(dbgs() << "[transGlobalAnnotation] " << *V << '\n'); // @llvm.global.annotations is an array that contains structs with 4 fields. // Get the array of structs with metadata // TODO: actually, now it contains 5 fields, the fifth by default is nullptr // or undef, but it can be defined to include variadic arguments of // clang::annotation attribute. Need to refactor this function to turn on this // translation ConstantArray *CA = cast(V->getOperand(0)); for (Value *Op : CA->operands()) { ConstantStruct *CS = cast(Op); // The first field of the struct contains a pointer to annotated variable Value *AnnotatedVar = CS->getOperand(0)->stripPointerCasts(); SPIRVValue *SV = transValue(AnnotatedVar, nullptr); // The second field contains a pointer to a global annotation string GlobalVariable *GV = cast(CS->getOperand(1)->stripPointerCasts()); StringRef AnnotationString; getConstantStringInfo(GV, AnnotationString); DecorationsInfoVec Decorations = tryParseAnnotationString(BM, AnnotationString).MemoryAttributesVec; // If we didn't find any annotation decorations, let's add the whole // annotation string as UserSemantic Decoration if (Decorations.empty()) { SV->addDecorate( new SPIRVDecorateUserSemanticAttr(SV, AnnotationString.str())); } else { addAnnotationDecorations(SV, Decorations); } } } void LLVMToSPIRVBase::transGlobalIOPipeStorage(GlobalVariable *V, MDNode *IO) { SPIRVDBG(dbgs() << "[transGlobalIOPipeStorage] " << *V << '\n'); SPIRVValue *SV = transValue(V, nullptr); assert(SV && "Failed to process OCL PipeStorage object"); if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_io_pipes)) { unsigned ID = getMDOperandAsInt(IO, 0); SV->addDecorate(DecorationIOPipeStorageINTEL, ID); } } bool LLVMToSPIRVBase::transGlobalVariables() { for (auto I = M->global_begin(), E = M->global_end(); I != E; ++I) { if ((*I).getName() == "llvm.global.annotations") transGlobalAnnotation(&(*I)); else if ([I]() -> bool { // Check if the GV is used only in var/ptr instructions. If yes - // skip processing of this since it's only an annotation GV. if (I->user_empty()) return false; for (auto *U : I->users()) { Value *V = U; while (isa(V) || isa(V)) V = cast(V)->getOperand(0); auto *GEP = dyn_cast_or_null(V); if (!GEP) return false; for (auto *GEPU : GEP->users()) { auto *II = dyn_cast(GEPU); if (!II) return false; switch (II->getIntrinsicID()) { case Intrinsic::var_annotation: case Intrinsic::ptr_annotation: continue; default: return false; } } } return true; }()) continue; else if ((I->getName() == "llvm.global_ctors" || I->getName() == "llvm.global_dtors") && !BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_function_pointers)) { // Function pointers are required to represent structor lists; do not // translate the variable if function pointers are not available. continue; } else if (MDNode *IO = ((*I).getMetadata("io_pipe_id"))) transGlobalIOPipeStorage(&(*I), IO); else if (!transValue(&(*I), nullptr)) return false; } return true; } bool LLVMToSPIRVBase::isAnyFunctionReachableFromFunction( const Function *FS, const std::unordered_set Funcs) const { std::unordered_set Done; std::unordered_set ToDo; ToDo.insert(FS); while (!ToDo.empty()) { auto It = ToDo.begin(); const Function *F = *It; if (Funcs.find(F) != Funcs.end()) return true; ToDo.erase(It); Done.insert(F); const CallGraphNode *FN = (*CG)[F]; for (unsigned I = 0; I < FN->size(); ++I) { const CallGraphNode *NN = (*FN)[I]; const Function *NNF = NN->getFunction(); if (!NNF) continue; if (Done.find(NNF) == Done.end()) { ToDo.insert(NNF); } } } return false; } std::vector LLVMToSPIRVBase::collectEntryPointInterfaces(SPIRVFunction *SF, Function *F) { std::vector Interface; for (auto &GV : M->globals()) { const auto AS = GV.getAddressSpace(); SPIRVModule *BM = SF->getModule(); if (!BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) if (AS != SPIRAS_Input && AS != SPIRAS_Output) continue; std::unordered_set Funcs; for (const auto &U : GV.uses()) { const Instruction *Inst = dyn_cast(U.getUser()); if (!Inst) continue; Funcs.insert(Inst->getFunction()); } if (isAnyFunctionReachableFromFunction(F, Funcs)) { SPIRVWord ModuleVersion = static_cast(BM->getSPIRVVersion()); if (AS != SPIRAS_Input && AS != SPIRAS_Output && ModuleVersion < static_cast(VersionNumber::SPIRV_1_4)) BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); Interface.push_back(ValueMap[&GV]->getId()); } } return Interface; } void LLVMToSPIRVBase::mutateFuncArgType( const std::map &ChangedType, Function *F) { for (auto &I : ChangedType) { for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; ++UI) { auto Call = dyn_cast(*UI); if (!Call) continue; auto Arg = Call->getArgOperand(I.first); auto OrigTy = Arg->getType(); if (OrigTy == I.second) continue; SPIRVDBG(dbgs() << "[mutate arg type] " << *Call << ", " << *Arg << '\n'); auto CastF = M->getOrInsertFunction(SPCV_CAST, I.second, OrigTy); std::vector Args; Args.push_back(Arg); auto Cast = CallInst::Create(CastF, Args, "", Call); Call->replaceUsesOfWith(Arg, Cast); SPIRVDBG(dbgs() << "[mutate arg type] -> " << *Cast << '\n'); } } } // Propagate contraction requirement of F up the call graph. void LLVMToSPIRVBase::fpContractUpdateRecursive(Function *F, FPContract FPC) { std::queue Users; for (User *FU : F->users()) { Users.push(FU); } bool EnableLogger = FPC == FPContract::DISABLED && !Users.empty(); if (EnableLogger) { SPIRVDBG(dbgs() << "[fp-contract] disabled for users of " << F->getName() << '\n'); } while (!Users.empty()) { User *U = Users.front(); Users.pop(); if (EnableLogger) { SPIRVDBG(dbgs() << "[fp-contract] user: " << *U << '\n'); } // Move from an Instruction to its Function if (Instruction *I = dyn_cast(U)) { Users.push(I->getFunction()); continue; } if (Function *F = dyn_cast(U)) { if (!joinFPContract(F, FPC)) { // FP contract was not updated - no need to propagate // This also terminates a recursion (if any). if (EnableLogger) { SPIRVDBG(dbgs() << "[fp-contract] already disabled " << F->getName() << '\n'); } continue; } if (EnableLogger) { SPIRVDBG(dbgs() << "[fp-contract] disabled for " << F->getName() << '\n'); } for (User *FU : F->users()) { Users.push(FU); } continue; } // Unwrap a constant until we reach an Instruction. // This is checked after the Function, because a Function is also a // Constant. if (Constant *C = dyn_cast(U)) { for (User *CU : C->users()) { Users.push(CU); } continue; } llvm_unreachable("Unexpected use."); } } void LLVMToSPIRVBase::transFunction(Function *I) { SPIRVFunction *BF = transFunctionDecl(I); // Creating all basic blocks before creating any instruction. for (auto &FI : *I) { transValue(&FI, nullptr); } for (auto &FI : *I) { SPIRVBasicBlock *BB = static_cast(transValue(&FI, nullptr)); for (auto &BI : FI) { transValue(&BI, BB, false); } } // Enable FP contraction unless proven otherwise joinFPContract(I, FPContract::ENABLED); fpContractUpdateRecursive(I, getFPContract(I)); if (isKernel(I)) { auto Interface = collectEntryPointInterfaces(BF, I); BM->addEntryPoint(ExecutionModelKernel, BF->getId(), BF->getName(), Interface); } } bool isEmptyLLVMModule(Module *M) { return M->empty() && // No functions M->global_empty(); // No global variables } bool LLVMToSPIRVBase::translate() { BM->setGeneratorVer(KTranslatorVer); if (isEmptyLLVMModule(M)) BM->addCapability(CapabilityLinkage); // Use the type scavenger to recover pointer element types. Scavenger = std::make_unique(*M); if (!lowerBuiltinCallsToVariables(M)) return false; if (!transSourceLanguage()) return false; if (!transExtension()) return false; if (!transBuiltinSet()) return false; if (!transAddressingMode()) return false; if (!transGlobalVariables()) return false; for (auto &F : *M) { auto FT = F.getFunctionType(); std::map ChangedType; oclGetMutatedArgumentTypesByBuiltin(FT, ChangedType, &F); mutateFuncArgType(ChangedType, &F); } // SPIR-V logical layout requires all function declarations go before // function definitions. std::vector Decls, Defs; for (auto &F : *M) { if (isBuiltinTransToInst(&F) || isBuiltinTransToExtInst(&F) || F.getName().startswith(SPCV_CAST) || F.getName().startswith(LLVM_MEMCPY) || F.getName().startswith(SAMPLER_INIT)) continue; if (F.isDeclaration()) Decls.push_back(&F); else Defs.push_back(&F); } for (auto I : Decls) transFunctionDecl(I); for (auto I : Defs) transFunction(I); if (!transMetadata()) return false; if (!transExecutionMode()) return false; BM->resolveUnknownStructFields(); DbgTran->transDebugMetadata(); return true; } llvm::IntegerType *LLVMToSPIRVBase::getSizetType(unsigned AS) { return IntegerType::getIntNTy(M->getContext(), M->getDataLayout().getPointerSizeInBits(AS)); } void LLVMToSPIRVBase::oclGetMutatedArgumentTypesByBuiltin( llvm::FunctionType *FT, std::map &ChangedType, Function *F) { StringRef Demangled; if (!oclIsBuiltin(F->getName(), Demangled)) return; // Note: kSPIRVName::ConvertHandleToSampledImageINTEL contains // kSPIRVName::SampledImage as a substring, but we still want to return in // this case. if (Demangled.find(kSPIRVName::SampledImage) == std::string::npos || Demangled.find(kSPIRVName::ConvertHandleToSampledImageINTEL) != std::string::npos) return; if (FT->getParamType(1)->isIntegerTy()) ChangedType[1] = getSamplerType(F->getParent()); } SPIRVValue *LLVMToSPIRVBase::transBuiltinToConstant(StringRef DemangledName, CallInst *CI) { Op OC = getSPIRVFuncOC(DemangledName); if (!isSpecConstantOpCode(OC)) return nullptr; if (OC == spv::OpSpecConstantComposite) { return BM->addSpecConstantComposite(transType(CI->getType()), transValue(getArguments(CI), nullptr)); } Value *V = CI->getArgOperand(1); Type *Ty = CI->getType(); assert(((Ty == V->getType()) || // If bool is stored into memory, then clang will emit it as i8, // however for other usages of bool (like return type of a function), // it is emitted as i1. // Therefore, situation when we encounter // i1 _Z20__spirv_SpecConstant(i32, i8) is valid (Ty->isIntegerTy(1) && V->getType()->isIntegerTy(8))) && "Type mismatch!"); uint64_t Val = 0; if (Ty->isIntegerTy()) Val = cast(V)->getZExtValue(); else if (Ty->isFloatingPointTy()) Val = cast(V)->getValueAPF().bitcastToAPInt().getZExtValue(); else return nullptr; SPIRVValue *SC = BM->addSpecConstant(transType(Ty), Val); return SC; } SPIRVInstruction *LLVMToSPIRVBase::transBuiltinToInst(StringRef DemangledName, CallInst *CI, SPIRVBasicBlock *BB) { SmallVector Dec; auto OC = getSPIRVFuncOC(DemangledName, &Dec); if (OC == OpNop) return nullptr; if (OpReadPipeBlockingINTEL <= OC && OC <= OpWritePipeBlockingINTEL && !BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_blocking_pipes)) return nullptr; if (OpFixedSqrtINTEL <= OC && OC <= OpFixedExpINTEL) BM->getErrorLog().checkError( BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_fixed_point), SPIRVEC_InvalidInstruction, CI->getCalledOperand()->getName().str() + "\nFixed point instructions can't be translated correctly without " "enabled SPV_INTEL_arbitrary_precision_fixed_point extension!\n"); if ((OpArbitraryFloatSinCosPiINTEL <= OC && OC <= OpArbitraryFloatCastToIntINTEL) || (OpArbitraryFloatAddINTEL <= OC && OC <= OpArbitraryFloatPowNINTEL)) BM->getErrorLog().checkError( BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_floating_point), SPIRVEC_InvalidInstruction, CI->getCalledOperand()->getName().str() + "\nFloating point instructions can't be translated correctly " "without enabled SPV_INTEL_arbitrary_precision_floating_point " "extension!\n"); auto Inst = transBuiltinToInstWithoutDecoration(OC, CI, BB); addDecorations(Inst, Dec); return Inst; } bool LLVMToSPIRVBase::transExecutionMode() { if (auto NMD = SPIRVMDWalker(*M).getNamedMD(kSPIRVMD::ExecutionMode)) { while (!NMD.atEnd()) { unsigned EMode = ~0U; Function *F = nullptr; auto N = NMD.nextOp(); /* execution mode MDNode */ N.get(F).get(EMode); SPIRVFunction *BF = static_cast(getTranslatedValue(F)); assert(BF && "Invalid kernel function"); if (!BF) return false; auto AddSingleArgExecutionMode = [&](ExecutionMode EMode) { uint32_t Arg = 0; N.get(Arg); BF->addExecutionMode( BM->add(new SPIRVExecutionMode(OpExecutionMode, BF, EMode, Arg))); }; switch (EMode) { case spv::ExecutionModeContractionOff: BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode)))); break; case spv::ExecutionModeInitializer: case spv::ExecutionModeFinalizer: if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_1)) { BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode)))); } else { getErrorLog().checkError(false, SPIRVEC_Requires1_1, "Initializer/Finalizer Execution Mode"); return false; } break; case spv::ExecutionModeLocalSize: case spv::ExecutionModeLocalSizeHint: { unsigned X = 0, Y = 0, Z = 0; N.get(X).get(Y).get(Z); BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode), X, Y, Z))); } break; case spv::ExecutionModeMaxWorkgroupSizeINTEL: { if (BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_kernel_attributes)) { unsigned X = 0, Y = 0, Z = 0; N.get(X).get(Y).get(Z); BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode), X, Y, Z))); BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes); BM->addCapability(CapabilityKernelAttributesINTEL); } } break; case spv::ExecutionModeNoGlobalOffsetINTEL: { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_kernel_attributes)) break; BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode)))); BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes); BM->addCapability(CapabilityKernelAttributesINTEL); } break; case spv::ExecutionModeVecTypeHint: case spv::ExecutionModeSubgroupSize: case spv::ExecutionModeSubgroupsPerWorkgroup: AddSingleArgExecutionMode(static_cast(EMode)); break; case spv::ExecutionModeNumSIMDWorkitemsINTEL: case spv::ExecutionModeSchedulerTargetFmaxMhzINTEL: case spv::ExecutionModeMaxWorkDimINTEL: case spv::ExecutionModeStreamingInterfaceINTEL: { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_kernel_attributes)) break; AddSingleArgExecutionMode(static_cast(EMode)); BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes); BM->addCapability(CapabilityFPGAKernelAttributesINTEL); } break; case spv::ExecutionModeSharedLocalMemorySizeINTEL: { if (!BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) break; AddSingleArgExecutionMode(static_cast(EMode)); } break; case spv::ExecutionModeNamedBarrierCountINTEL: { if (!BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) break; unsigned NBarrierCnt = 0; N.get(NBarrierCnt); BF->addExecutionMode(BM->add(new SPIRVExecutionMode( OpExecutionMode, BF, static_cast(EMode), NBarrierCnt))); BM->addExtension(ExtensionID::SPV_INTEL_vector_compute); BM->addCapability(CapabilityVectorComputeINTEL); } break; case spv::ExecutionModeDenormPreserve: case spv::ExecutionModeDenormFlushToZero: case spv::ExecutionModeSignedZeroInfNanPreserve: case spv::ExecutionModeRoundingModeRTE: case spv::ExecutionModeRoundingModeRTZ: { if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) { BM->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); AddSingleArgExecutionMode(static_cast(EMode)); } else if (BM->isAllowedToUseExtension( ExtensionID::SPV_KHR_float_controls)) { BM->addExtension(ExtensionID::SPV_KHR_float_controls); AddSingleArgExecutionMode(static_cast(EMode)); } } break; case spv::ExecutionModeRoundingModeRTPINTEL: case spv::ExecutionModeRoundingModeRTNINTEL: case spv::ExecutionModeFloatingPointModeALTINTEL: case spv::ExecutionModeFloatingPointModeIEEEINTEL: { if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_float_controls2)) break; AddSingleArgExecutionMode(static_cast(EMode)); } break; default: llvm_unreachable("invalid execution mode"); } } } transFPContract(); return true; } void LLVMToSPIRVBase::transFPContract() { FPContractMode Mode = BM->getFPContractMode(); for (Function &F : *M) { SPIRVValue *TranslatedF = getTranslatedValue(&F); if (!TranslatedF) { continue; } SPIRVFunction *BF = static_cast(TranslatedF); bool IsKernelEntryPoint = BF->getModule()->isEntryPoint(spv::ExecutionModelKernel, BF->getId()); if (!IsKernelEntryPoint) continue; FPContract FPC = getFPContract(&F); assert(FPC != FPContract::UNDEF); bool DisableContraction = false; switch (Mode) { case FPContractMode::Fast: DisableContraction = false; break; case FPContractMode::On: DisableContraction = FPC == FPContract::DISABLED; break; case FPContractMode::Off: DisableContraction = true; break; } if (DisableContraction) { BF->addExecutionMode(BF->getModule()->add(new SPIRVExecutionMode( OpExecutionMode, BF, spv::ExecutionModeContractionOff))); } } } bool LLVMToSPIRVBase::transMetadata() { if (!transOCLMetadata()) return false; auto Model = getMemoryModel(*M); if (Model != SPIRVMemoryModelKind::MemoryModelMax) BM->setMemoryModel(static_cast(Model)); return true; } // Work around to translate kernel_arg_type and kernel_arg_type_qual metadata static void transKernelArgTypeMD(SPIRVModule *BM, Function *F, MDNode *MD, std::string MDName) { std::string Prefix = kSPIRVName::EntrypointPrefix; std::string Name = F->getName().str().substr(Prefix.size()); std::string KernelArgTypesMDStr = std::string(MDName) + "." + Name + "."; for (const auto &TyOp : MD->operands()) KernelArgTypesMDStr += cast(TyOp)->getString().str() + ","; BM->getString(KernelArgTypesMDStr); } bool LLVMToSPIRVBase::transOCLMetadata() { for (auto &F : *M) { if (F.getCallingConv() != CallingConv::SPIR_KERNEL) continue; SPIRVFunction *BF = static_cast(getTranslatedValue(&F)); assert(BF && "Kernel function should be translated first"); // Create 'OpString' as a workaround to store information about // *orignal* (typedef'ed, unsigned integers) type names of kernel arguments. // OpString "kernel_arg_type.%kernel_name%.typename0,typename1,..." if (auto *KernelArgType = F.getMetadata(SPIR_MD_KERNEL_ARG_TYPE)) if (BM->shouldPreserveOCLKernelArgTypeMetadataThroughString()) transKernelArgTypeMD(BM, &F, KernelArgType, SPIR_MD_KERNEL_ARG_TYPE); if (auto *KernelArgTypeQual = F.getMetadata(SPIR_MD_KERNEL_ARG_TYPE_QUAL)) { foreachKernelArgMD( KernelArgTypeQual, BF, [](const std::string &Str, SPIRVFunctionParameter *BA) { if (Str.find("volatile") != std::string::npos) BA->addDecorate(new SPIRVDecorate(DecorationVolatile, BA)); if (Str.find("restrict") != std::string::npos) BA->addDecorate( new SPIRVDecorate(DecorationFuncParamAttr, BA, FunctionParameterAttributeNoAlias)); }); // Create 'OpString' as a workaround to store information about // constant qualifiers of pointer kernel arguments. Store empty string // for a non constant parameter. // OpString "kernel_arg_type_qual.%kernel_name%.qual0,qual1,..." if (BM->shouldPreserveOCLKernelArgTypeMetadataThroughString()) transKernelArgTypeMD(BM, &F, KernelArgTypeQual, SPIR_MD_KERNEL_ARG_TYPE_QUAL); } if (auto *KernelArgName = F.getMetadata(SPIR_MD_KERNEL_ARG_NAME)) { foreachKernelArgMD( KernelArgName, BF, [=](const std::string &Str, SPIRVFunctionParameter *BA) { BM->setName(BA, Str); }); } if (auto *KernArgDecoMD = F.getMetadata(SPIRV_MD_PARAMETER_DECORATIONS)) foreachKernelArgMD(KernArgDecoMD, BF, transMetadataDecorations); } return true; } bool LLVMToSPIRVBase::transSourceLanguage() { auto Src = getSPIRVSource(M); SrcLang = std::get<0>(Src); SrcLangVer = std::get<1>(Src); BM->setSourceLanguage(static_cast(SrcLang), SrcLangVer); return true; } bool LLVMToSPIRVBase::transExtension() { if (auto N = SPIRVMDWalker(*M).getNamedMD(kSPIRVMD::Extension)) { while (!N.atEnd()) { std::string S; N.nextOp().get(S); assert(!S.empty() && "Invalid extension"); BM->getExtension().insert(S); } } if (auto N = SPIRVMDWalker(*M).getNamedMD(kSPIRVMD::SourceExtension)) { while (!N.atEnd()) { std::string S; N.nextOp().get(S); assert(!S.empty() && "Invalid extension"); BM->getSourceExtension().insert(S); } } for (auto &I : map(rmap(BM->getExtension()))) BM->addCapability(I); return true; } void LLVMToSPIRVBase::dumpUsers(Value *V) { SPIRVDBG(dbgs() << "Users of " << *V << " :\n"); for (auto UI = V->user_begin(), UE = V->user_end(); UI != UE; ++UI) SPIRVDBG(dbgs() << " " << **UI << '\n'); } Op LLVMToSPIRVBase::transBoolOpCode(SPIRVValue *Opn, Op OC) { if (!Opn->getType()->isTypeVectorOrScalarBool()) return OC; IntBoolOpMap::find(OC, &OC); return OC; } SPIRVInstruction * LLVMToSPIRVBase::transBuiltinToInstWithoutDecoration(Op OC, CallInst *CI, SPIRVBasicBlock *BB) { // We should do this replacement only for SPIR-V 1.5, as OpLessOrGreater is // deprecated there. However we do such replacement for the usual pipeline // (not via SPIR-V friendly calls) without minding the version, so we can do // such thing here as well. if (OC == OpLessOrGreater && BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_5)) OC = OpFOrdNotEqual; if (isGroupOpCode(OC)) BM->addCapability(CapabilityGroups); switch (OC) { case OpControlBarrier: { auto BArgs = transValue(getArguments(CI), BB); return BM->addControlBarrierInst(BArgs[0], BArgs[1], BArgs[2], BB); } break; case OpGroupAsyncCopy: { auto BArgs = transValue(getArguments(CI), BB); return BM->addAsyncGroupCopy(BArgs[0], BArgs[1], BArgs[2], BArgs[3], BArgs[4], BArgs[5], BB); } break; case OpVectorExtractDynamic: { auto BArgs = transValue(getArguments(CI), BB); return BM->addVectorExtractDynamicInst(BArgs[0], BArgs[1], BB); } break; case OpVectorInsertDynamic: { auto BArgs = transValue(getArguments(CI), BB); return BM->addVectorInsertDynamicInst(BArgs[0], BArgs[1], BArgs[2], BB); } break; case OpSampledImage: { // Clang can generate SPIRV-friendly call for OpSampledImage instruction, // i.e. __spirv_SampledImage... But it can't generate correct return type // for this call, because there is no support for type corresponding to // OpTypeSampledImage. So, in this case, we create the required type here. Value *Image = CI->getArgOperand(0); SmallVector ParamTys; getParameterTypes(CI, ParamTys); Type *ImageTy = adaptSPIRVImageType(M, ParamTys[0]); Type *SampledImgTy = getSPIRVStructTypeByChangeBaseTypeName( M, ImageTy, kSPIRVTypeName::Image, kSPIRVTypeName::SampledImg); Value *Sampler = CI->getArgOperand(1); return BM->addSampledImageInst( transPointerType(SampledImgTy, SPIRAS_Global), transValue(Image, BB), transValue(Sampler, BB), BB); } case OpFixedSqrtINTEL: case OpFixedRecipINTEL: case OpFixedRsqrtINTEL: case OpFixedSinINTEL: case OpFixedCosINTEL: case OpFixedSinCosINTEL: case OpFixedSinPiINTEL: case OpFixedCosPiINTEL: case OpFixedSinCosPiINTEL: case OpFixedLogINTEL: case OpFixedExpINTEL: { // LLVM fixed point functions return value: // iN (arbitrary precision integer of N bits length) // Arguments: // A(iN), S(i1), I(i32), rI(i32), Quantization(i32), Overflow(i32) // where A - integer input of any width. // SPIR-V fixed point instruction contains: // ResTy Res In \ // Literal S Literal I Literal rI Literal Q Literal O Type *ResTy = CI->getType(); auto OpItr = CI->value_op_begin(); auto OpEnd = OpItr + CI->arg_size(); // If the return type of an instruction is wider than 64-bit, then this // instruction will return via 'sret' argument added into the arguments // list. Here we reverse this, removing 'sret' argument and restoring // the original return type. if (CI->hasStructRetAttr()) { assert(ResTy->isVoidTy() && "Return type is not void"); ResTy = CI->getParamStructRetType(0); OpItr++; } // Input - integer input of any width or 'byval' pointer to this integer SPIRVValue *Input = transValue(*OpItr, BB); if (OpItr->getType()->isPointerTy()) Input = BM->addLoadInst(Input, {}, BB); OpItr++; std::vector Literals; std::transform(OpItr, OpEnd, std::back_inserter(Literals), [](auto *O) { return cast(O)->getZExtValue(); }); auto *APIntInst = BM->addFixedPointIntelInst(OC, transType(ResTy), Input, Literals, BB); if (!CI->hasStructRetAttr()) return APIntInst; return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), APIntInst, {}, BB); } case OpArbitraryFloatCastINTEL: case OpArbitraryFloatCastFromIntINTEL: case OpArbitraryFloatCastToIntINTEL: case OpArbitraryFloatRecipINTEL: case OpArbitraryFloatRSqrtINTEL: case OpArbitraryFloatCbrtINTEL: case OpArbitraryFloatSqrtINTEL: case OpArbitraryFloatLogINTEL: case OpArbitraryFloatLog2INTEL: case OpArbitraryFloatLog10INTEL: case OpArbitraryFloatLog1pINTEL: case OpArbitraryFloatExpINTEL: case OpArbitraryFloatExp2INTEL: case OpArbitraryFloatExp10INTEL: case OpArbitraryFloatExpm1INTEL: case OpArbitraryFloatSinINTEL: case OpArbitraryFloatCosINTEL: case OpArbitraryFloatSinCosINTEL: case OpArbitraryFloatSinPiINTEL: case OpArbitraryFloatCosPiINTEL: case OpArbitraryFloatSinCosPiINTEL: case OpArbitraryFloatASinINTEL: case OpArbitraryFloatASinPiINTEL: case OpArbitraryFloatACosINTEL: case OpArbitraryFloatACosPiINTEL: case OpArbitraryFloatATanINTEL: case OpArbitraryFloatATanPiINTEL: { // Format of instruction CastFromInt: // LLVM arbitrary floating point functions return value type: // iN (arbitrary precision integer of N bits length) // Arguments: A(iN), Mout(i32), FromSign(bool), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal Mout Literal FromSign // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instruction CastToInt: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), ToSign(bool), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA Literal ToSign // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of other instructions: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), Mout(i32), EnableSubnormals(i32), // RoundingMode(i32), RoundingAccuracy(i32) // where A and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA Literal Mout // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy Type *ResTy = CI->getType(); auto OpItr = CI->value_op_begin(); auto OpEnd = OpItr + CI->arg_size(); // If the return type of an instruction is wider than 64-bit, then this // instruction will return via 'sret' argument added into the arguments // list. Here we reverse this, removing 'sret' argument and restoring // the original return type. if (CI->hasStructRetAttr()) { assert(ResTy->isVoidTy() && "Return type is not void"); ResTy = CI->getParamStructRetType(0); OpItr++; } // Input - integer input of any width or 'byval' pointer to this integer SPIRVValue *Input = transValue(*OpItr, BB); if (OpItr->getType()->isPointerTy()) Input = BM->addLoadInst(Input, {}, BB); OpItr++; std::vector Literals; std::transform(OpItr, OpEnd, std::back_inserter(Literals), [](auto *O) { return cast(O)->getZExtValue(); }); auto *APIntInst = BM->addArbFloatPointIntelInst(OC, transType(ResTy), Input, nullptr, Literals, BB); if (!CI->hasStructRetAttr()) return APIntInst; return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), APIntInst, {}, BB); } case OpArbitraryFloatAddINTEL: case OpArbitraryFloatSubINTEL: case OpArbitraryFloatMulINTEL: case OpArbitraryFloatDivINTEL: case OpArbitraryFloatGTINTEL: case OpArbitraryFloatGEINTEL: case OpArbitraryFloatLTINTEL: case OpArbitraryFloatLEINTEL: case OpArbitraryFloatEQINTEL: case OpArbitraryFloatHypotINTEL: case OpArbitraryFloatATan2INTEL: case OpArbitraryFloatPowINTEL: case OpArbitraryFloatPowRINTEL: case OpArbitraryFloatPowNINTEL: { // Format of instructions Add, Sub, Mul, Div, Hypot, ATan2, Pow, PowR: // LLVM arbitrary floating point functions return value: // iN (arbitrary precision integer of N bits length) // Arguments: A(iN), MA(i32), B(iN), MB(i32), Mout(i32), // EnableSubnormals(i32), RoundingMode(i32), // RoundingAccuracy(i32) // where A, B and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal MB Literal Mout // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instruction PowN: // LLVM arbitrary floating point functions return value: iN // Arguments: A(iN), MA(i32), B(iN), SignOfB(i1), Mout(i32), // EnableSubnormals(i32), RoundingMode(i32), // RoundingAccuracy(i32) // where A, B and return values are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal SignOfB Literal Mout // Literal EnableSubnormals Literal RoundingMode // Literal RoundingAccuracy // Format of instructions GT, GE, LT, LE, EQ: // LLVM arbitrary floating point functions return value: Bool // Arguments: A(iN), MA(i32), B(iN), MB(i32) // where A and B are of arbitrary precision integer type. // SPIR-V arbitrary floating point instruction layout: // ResTy Res A Literal MA B Literal MB Type *ResTy = CI->getType(); auto OpItr = CI->value_op_begin(); auto OpEnd = OpItr + CI->arg_size(); // If the return type of an instruction is wider than 64-bit, then this // instruction will return via 'sret' argument added into the arguments // list. Here we reverse this, removing 'sret' argument and restoring // the original return type. if (CI->hasStructRetAttr()) { assert(ResTy->isVoidTy() && "Return type is not void"); ResTy = CI->getParamStructRetType(0); OpItr++; } // InA - integer input of any width or 'byval' pointer to this integer SPIRVValue *InA = transValue(*OpItr, BB); if (OpItr->getType()->isPointerTy()) InA = BM->addLoadInst(InA, {}, BB); OpItr++; std::vector Literals; Literals.push_back(cast(*OpItr++)->getZExtValue()); // InB - integer input of any width or 'byval' pointer to this integer SPIRVValue *InB = transValue(*OpItr, BB); if (OpItr->getType()->isPointerTy()) { std::vector Mem; InB = BM->addLoadInst(InB, Mem, BB); } OpItr++; std::transform(OpItr, OpEnd, std::back_inserter(Literals), [](auto *O) { return cast(O)->getZExtValue(); }); auto *APIntInst = BM->addArbFloatPointIntelInst(OC, transType(ResTy), InA, InB, Literals, BB); if (!CI->hasStructRetAttr()) return APIntInst; return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), APIntInst, {}, BB); } case OpLoad: { std::vector MemoryAccess; assert(CI->arg_size() > 0 && "Expected at least 1 operand for OpLoad call"); for (size_t I = 1; I < CI->arg_size(); ++I) MemoryAccess.push_back( cast(CI->getArgOperand(I))->getZExtValue()); return BM->addLoadInst(transValue(CI->getArgOperand(0), BB), MemoryAccess, BB); } case OpStore: { std::vector MemoryAccess; assert(CI->arg_size() > 1 && "Expected at least 2 operands for OpStore call"); for (size_t I = 2; I < CI->arg_size(); ++I) MemoryAccess.push_back( cast(CI->getArgOperand(I))->getZExtValue()); return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), transValue(CI->getArgOperand(1), BB), MemoryAccess, BB); } case OpCompositeConstruct: { std::vector Operands = { transValue(CI->getArgOperand(0), BB)->getId()}; return BM->addCompositeConstructInst(transType(CI->getType()), Operands, BB); } case OpIAddCarry: case OpISubBorrow: case OpUMulExtended: case OpSMulExtended: { Function *F = CI->getCalledFunction(); StructType *St = cast(F->getParamStructRetType(0)); SPIRVValue *V = BM->addBinaryInst(OC, transType(St), transValue(CI->getArgOperand(1), BB), transValue(CI->getArgOperand(2), BB), BB); return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), V, {}, BB); } case OpCooperativeMatrixLengthKHR: { return BM->addCooperativeMatrixLengthKHRInst( transScavengedType(CI), transType(CI->getArgOperand(0)->getType()), BB); } default: { if (isCvtOpCode(OC) && OC != OpGenericCastToPtrExplicit) { return BM->addUnaryInst(OC, transType(CI->getType()), transValue(CI->getArgOperand(0), BB), BB); } else if (isCmpOpCode(OC) || isUnaryPredicateOpCode(OC)) { auto ResultTy = CI->getType(); Type *BoolTy = IntegerType::getInt1Ty(M->getContext()); auto IsVector = ResultTy->isVectorTy(); if (IsVector) BoolTy = FixedVectorType::get( BoolTy, cast(ResultTy)->getNumElements()); auto BBT = transType(BoolTy); SPIRVInstruction *Res; if (isCmpOpCode(OC)) { assert(CI && CI->arg_size() == 2 && "Invalid call inst"); Res = BM->addCmpInst(OC, BBT, transValue(CI->getArgOperand(0), BB), transValue(CI->getArgOperand(1), BB), BB); } else { assert(CI && CI->arg_size() == 1 && "Invalid call inst"); Res = BM->addUnaryInst(OC, BBT, transValue(CI->getArgOperand(0), BB), BB); } // OpenCL C and OpenCL C++ built-ins may have different return type if (ResultTy == BoolTy) return Res; assert(IsVector || (!IsVector && ResultTy->isIntegerTy(32))); auto Zero = transValue(Constant::getNullValue(ResultTy), BB); auto One = transValue( IsVector ? Constant::getAllOnesValue(ResultTy) : getInt32(M, 1), BB); return BM->addSelectInst(Res, One, Zero, BB); } else if (isBinaryOpCode(OC)) { assert(CI && CI->arg_size() == 2 && "Invalid call inst"); return BM->addBinaryInst(OC, transType(CI->getType()), transValue(CI->getArgOperand(0), BB), transValue(CI->getArgOperand(1), BB), BB); } else if (CI->arg_size() == 1 && !CI->getType()->isVoidTy() && !hasExecScope(OC) && !isAtomicOpCode(OC)) { return BM->addUnaryInst(OC, transType(CI->getType()), transValue(CI->getArgOperand(0), BB), BB); } else { auto Args = getArguments(CI); SPIRVType *SPRetTy = nullptr; Type *RetTy = CI->getType(); auto F = CI->getCalledFunction(); if (!RetTy->isVoidTy()) { SPRetTy = transType(RetTy); } else if (Args.size() > 0 && F->arg_begin()->hasStructRetAttr()) { SPRetTy = transType(F->getParamStructRetType(0)); Args.erase(Args.begin()); } auto *SPI = SPIRVInstTemplateBase::create(OC); std::vector SPArgs; for (size_t I = 0, E = Args.size(); I != E; ++I) { if (Args[I]->getType()->isPointerTy()) { Value *Pointee = Args[I]->stripPointerCasts(); (void)Pointee; assert((Pointee == Args[I] || !isa(Pointee)) && "Illegal use of a function pointer type"); } SPArgs.push_back(SPI->isOperandLiteral(I) ? cast(Args[I])->getZExtValue() : transValue(Args[I], BB)->getId()); } BM->addInstTemplate(SPI, SPArgs, BB, SPRetTy); if (!SPRetTy || !SPRetTy->isTypeStruct()) return SPI; std::vector Mem; SPIRVDBG(spvdbgs() << *SPI << '\n'); return BM->addStoreInst(transValue(CI->getArgOperand(0), BB), SPI, Mem, BB); } } } return nullptr; } SPIRV::SPIRVLinkageTypeKind LLVMToSPIRVBase::transLinkageType(const GlobalValue *GV) { if (GV->isDeclarationForLinker()) return SPIRVLinkageTypeKind::LinkageTypeImport; if (GV->hasInternalLinkage() || GV->hasPrivateLinkage()) return spv::internal::LinkageTypeInternal; if (GV->hasLinkOnceODRLinkage()) if (BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_linkonce_odr)) return SPIRVLinkageTypeKind::LinkageTypeLinkOnceODR; return SPIRVLinkageTypeKind::LinkageTypeExport; } LLVMToSPIRVBase::FPContract LLVMToSPIRVBase::getFPContract(Function *F) { auto It = FPContractMap.find(F); if (It == FPContractMap.end()) { return FPContract::UNDEF; } return It->second; } bool LLVMToSPIRVBase::joinFPContract(Function *F, FPContract C) { FPContract &Existing = FPContractMap[F]; switch (Existing) { case FPContract::UNDEF: if (C != FPContract::UNDEF) { Existing = C; return true; } return false; case FPContract::ENABLED: if (C == FPContract::DISABLED) { Existing = C; return true; } return false; case FPContract::DISABLED: return false; } llvm_unreachable("Unhandled FPContract value."); } } // namespace SPIRV char LLVMToSPIRVLegacy::ID = 0; INITIALIZE_PASS_BEGIN(LLVMToSPIRVLegacy, "llvmtospv", "Translate LLVM to SPIR-V", false, false) INITIALIZE_PASS_DEPENDENCY(OCLTypeToSPIRVLegacy) INITIALIZE_PASS_END(LLVMToSPIRVLegacy, "llvmtospv", "Translate LLVM to SPIR-V", false, false) ModulePass *llvm::createLLVMToSPIRVLegacy(SPIRVModule *SMod) { return new LLVMToSPIRVLegacy(SMod); } void addPassesForSPIRV(ModulePassManager &PassMgr, const SPIRV::TranslatorOpts &Opts) { if (Opts.isSPIRVMemToRegEnabled()) PassMgr.addPass(createModuleToFunctionPassAdaptor(PromotePass())); PassMgr.addPass(PreprocessMetadataPass()); PassMgr.addPass(SPIRVLowerOCLBlocksPass()); PassMgr.addPass(OCLToSPIRVPass()); PassMgr.addPass(SPIRVRegularizeLLVMPass()); PassMgr.addPass(SPIRVLowerConstExprPass()); PassMgr.addPass(SPIRVLowerBoolPass()); PassMgr.addPass(SPIRVLowerMemmovePass()); PassMgr.addPass(SPIRVLowerSaddIntrinsicsPass()); PassMgr.addPass(createModuleToFunctionPassAdaptor( SPIRVLowerBitCastToNonStandardTypePass(Opts))); } bool isValidLLVMModule(Module *M, SPIRVErrorLog &ErrorLog) { if (!M) return false; if (isEmptyLLVMModule(M)) return true; Triple TT(M->getTargetTriple()); if (!ErrorLog.checkError(isSupportedTriple(TT), SPIRVEC_InvalidTargetTriple, "Actual target triple is " + M->getTargetTriple())) return false; return true; } namespace { VersionNumber getVersionFromTriple(const Triple &TT, SPIRVErrorLog &ErrorLog) { switch (TT.getSubArch()) { case Triple::SPIRVSubArch_v10: return VersionNumber::SPIRV_1_0; case Triple::SPIRVSubArch_v11: return VersionNumber::SPIRV_1_1; case Triple::SPIRVSubArch_v12: return VersionNumber::SPIRV_1_2; case Triple::SPIRVSubArch_v13: return VersionNumber::SPIRV_1_3; case Triple::SPIRVSubArch_v14: return VersionNumber::SPIRV_1_4; case Triple::SPIRVSubArch_v15: return VersionNumber::SPIRV_1_5; default: ErrorLog.checkError(false, SPIRVEC_InvalidSubArch, TT.getArchName().str()); return VersionNumber::MaximumVersion; } } bool runSpirvWriterPasses(Module *M, std::ostream *OS, std::string &ErrMsg, const SPIRV::TranslatorOpts &Opts) { // Perform the conversion and write the resulting SPIR-V if an ostream has // been given; otherwise only perform regularization. bool WriteSpirv = OS != nullptr; std::unique_ptr BM(SPIRVModule::createSPIRVModule(Opts)); if (!isValidLLVMModule(M, BM->getErrorLog())) return false; // If the module carries a SPIR-V triple with a version subarch, target // that SPIR-V version. Triple TargetTriple(M->getTargetTriple()); if ((TargetTriple.getArch() == Triple::spirv32 || TargetTriple.getArch() == Triple::spirv64) && TargetTriple.getSubArch() != Triple::NoSubArch) { VersionNumber ModuleVer = getVersionFromTriple(TargetTriple, BM->getErrorLog()); if (!BM->getErrorLog().checkError(ModuleVer <= Opts.getMaxVersion(), SPIRVEC_TripleMaxVersionIncompatible)) return false; BM->setMinSPIRVVersion(ModuleVer); } ModulePassManager PassMgr; addPassesForSPIRV(PassMgr, Opts); if (WriteSpirv) { // Run loop simplify pass in order to avoid duplicate OpLoopMerge // instruction. It can happen in case of continue operand in the loop. if (hasLoopMetadata(M)) PassMgr.addPass(createModuleToFunctionPassAdaptor(LoopSimplifyPass())); PassMgr.addPass(LLVMToSPIRVPass(BM.get())); } LoopAnalysisManager LAM; CGSCCAnalysisManager CGAM; FunctionAnalysisManager FAM; ModuleAnalysisManager MAM; PassBuilder PB; PB.registerModuleAnalyses(MAM); PB.registerCGSCCAnalyses(CGAM); PB.registerFunctionAnalyses(FAM); PB.registerLoopAnalyses(LAM); MAM.registerPass([&] { return OCLTypeToSPIRVPass(); }); PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); PassMgr.run(*M, MAM); if (BM->getError(ErrMsg) != SPIRVEC_Success) return false; if (WriteSpirv) *OS << *BM; return true; } } // namespace bool llvm::writeSpirv(Module *M, std::ostream &OS, std::string &ErrMsg) { SPIRV::TranslatorOpts DefaultOpts; // To preserve old behavior of the translator, let's enable all extensions // by default in this API DefaultOpts.enableAllExtensions(); return llvm::writeSpirv(M, DefaultOpts, OS, ErrMsg); } bool llvm::writeSpirv(Module *M, const SPIRV::TranslatorOpts &Opts, std::ostream &OS, std::string &ErrMsg) { return runSpirvWriterPasses(M, &OS, ErrMsg, Opts); } bool llvm::regularizeLlvmForSpirv(Module *M, std::string &ErrMsg) { SPIRV::TranslatorOpts DefaultOpts; // To preserve old behavior of the translator, let's enable all extensions // by default in this API DefaultOpts.enableAllExtensions(); return llvm::regularizeLlvmForSpirv(M, ErrMsg, DefaultOpts); } bool llvm::regularizeLlvmForSpirv(Module *M, std::string &ErrMsg, const SPIRV::TranslatorOpts &Opts) { return runSpirvWriterPasses(M, nullptr, ErrMsg, Opts); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVWriter.h000066400000000000000000000303471506646577400214440ustar00rootroot00000000000000//===- SPIRVWriter.h - Converts LLVM to SPIR-V ------------------*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file contains declaration of LLVMToSPIRV class which implements /// conversion of LLVM intermediate language to SPIR-V /// binary. /// //===----------------------------------------------------------------------===// #ifndef SPIRVWRITER_H #define SPIRVWRITER_H #include "LLVMToSPIRVDbgTran.h" #include "OCLTypeToSPIRV.h" #include "OCLUtil.h" #include "SPIRVBasicBlock.h" #include "SPIRVEntry.h" #include "SPIRVEnum.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVModule.h" #include "SPIRVType.h" #include "SPIRVTypeScavenger.h" #include "SPIRVValue.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/IR/IntrinsicInst.h" #include using namespace llvm; using namespace SPIRV; using namespace OCLUtil; namespace SPIRV { class LLVMToSPIRVBase { public: LLVMToSPIRVBase(SPIRVModule *SMod); bool runLLVMToSPIRV(Module &Mod); // This enum sets the mode used to translate the value which is // a function, that is necessary for a convenient function pointers handling. // By default transValue uses 'Decl' mode, which means every function // we meet during the translation should result in its declaration generated. // In 'Pointer' mode we generate OpConstantFunctionPointerINTEL constant // instead. enum class FuncTransMode { Decl, Pointer }; SPIRVType *transType(Type *T); SPIRVType *transPointerType(Type *PointeeTy, unsigned AddrSpace); SPIRVType *transPointerType(SPIRVType *PointeeTy, unsigned AddrSpace); SPIRVType *transSPIRVOpaqueType(StringRef STName, unsigned AddrSpace); SPIRVType * transSPIRVJointOrCooperativeMatrixType(SmallVector Postfixes, bool IsCooperative = false); // SPIRVType* transSPIRVCooperativeMatrixKHRType(); /// Use the type scavenger to get the correct type for V. This is equivalent /// to transType(V->getType()) if V is not a pointer type; otherwise, it tries /// to pick an appropriate pointee type for V. SPIRVType *transScavengedType(Value *V); SPIRVValue *getTranslatedValue(const Value *) const; spv::LoopControlMask getLoopControl(const BranchInst *Branch, std::vector &Parameters); // Translation functions bool transAddressingMode(); bool transAlign(Value *V, SPIRVValue *BV); std::vector transArguments(CallInst *, SPIRVBasicBlock *, SPIRVEntry *); bool transSourceLanguage(); bool transExtension(); bool transBuiltinSet(); bool isKnownIntrinsic(Intrinsic::ID Id); SPIRVValue *transIntrinsicInst(IntrinsicInst *Intrinsic, SPIRVBasicBlock *BB); enum class FPBuiltinType { REGULAR_MATH, EXT_1OPS, EXT_2OPS, EXT_3OPS, UNKNOWN }; FPBuiltinType getFPBuiltinType(IntrinsicInst *II, StringRef &); SPIRVValue *transFPBuiltinIntrinsicInst(IntrinsicInst *II, SPIRVBasicBlock *BB); SPIRVValue *transFenceInst(FenceInst *FI, SPIRVBasicBlock *BB); SPIRVValue *transCallInst(CallInst *Call, SPIRVBasicBlock *BB); SPIRVValue *transDirectCallInst(CallInst *Call, SPIRVBasicBlock *BB); SPIRVValue *transIndirectCallInst(CallInst *Call, SPIRVBasicBlock *BB); SPIRVValue *transAsmINTEL(InlineAsm *Asm); SPIRVValue *transAsmCallINTEL(CallInst *Call, SPIRVBasicBlock *BB); bool transDecoration(Value *V, SPIRVValue *BV); bool shouldTryToAddMemAliasingDecoration(Instruction *V); void transMemAliasingINTELDecorations(Instruction *V, SPIRVValue *BV); SPIRVWord transFunctionControlMask(Function *); SPIRVFunction *transFunctionDecl(Function *F); void transVectorComputeMetadata(Function *F); void transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F); void transAuxDataInst(SPIRVFunction *BF, Function *F); void transFunctionMetadataAsExecutionMode(SPIRVFunction *BF, Function *F); bool transGlobalVariables(); Op transBoolOpCode(SPIRVValue *Opn, Op OC); // Translate LLVM module to SPIR-V module. // Returns true if succeeds. bool translate(); bool transExecutionMode(); void transFPContract(); SPIRVValue *transConstant(Value *V); SPIRVValue *transValue(Value *V, SPIRVBasicBlock *BB, bool CreateForward = true, FuncTransMode FuncTrans = FuncTransMode::Decl); void transGlobalAnnotation(GlobalVariable *V); SPIRVValue * transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, bool CreateForward = true, FuncTransMode FuncTrans = FuncTransMode::Decl); void transGlobalIOPipeStorage(GlobalVariable *V, MDNode *IO); static SPIRVInstruction *applyRoundingModeConstraint(Value *V, SPIRVInstruction *I); typedef DenseMap LLVMToSPIRVTypeMap; typedef DenseMap LLVMToSPIRVValueMap; typedef DenseMap> LLVMToSPIRVMetadataMap; void setOCLTypeToSPIRV(OCLTypeToSPIRVBase *OCLTypeToSPIRV) { OCLTypeToSPIRVPtr = OCLTypeToSPIRV; } OCLTypeToSPIRVBase *getOCLTypeToSPIRV() { return OCLTypeToSPIRVPtr; } ~LLVMToSPIRVBase(); private: Module *M; LLVMContext *Ctx; SPIRVModule *BM; // This maps LLVM types (except for pointers) to SPIRVType. LLVMToSPIRVTypeMap TypeMap; // This maps {struct name, addrspace} to SPIRVType, for those structs that // represent special SPIRV types. DenseMap, SPIRVType *> OpaqueStructMap; // This maps to SPIRVType, for use in function types. StringMap PointeeTypeMap; /// Get the SPIRVFunctionType with appropriate return and argument types, /// returning an existing instance if one has already been created. This is /// necessary to unique locally, as SPIRVModule does not do such uniquing. SPIRVType *getSPIRVFunctionType(SPIRVType *RT, const std::vector &Args); LLVMToSPIRVValueMap ValueMap; LLVMToSPIRVMetadataMap IndexGroupArrayMap; SPIRVWord SrcLang; SPIRVWord SrcLangVer; std::unique_ptr DbgTran; std::unique_ptr CG; OCLTypeToSPIRVBase *OCLTypeToSPIRVPtr; std::vector UnboundInst; std::unique_ptr Scavenger; enum class FPContract { UNDEF, DISABLED, ENABLED }; DenseMap FPContractMap; FPContract getFPContract(Function *F); bool joinFPContract(Function *F, FPContract C); void fpContractUpdateRecursive(Function *F, FPContract FPC); SPIRVType *mapType(Type *T, SPIRVType *BT); SPIRVValue *mapValue(Value *V, SPIRVValue *BV); SPIRVErrorLog &getErrorLog() { return BM->getErrorLog(); } llvm::IntegerType *getSizetType(unsigned AS = 0); std::vector transValue(const std::vector &Values, SPIRVBasicBlock *BB); std::vector transValue(const std::vector &Values, SPIRVBasicBlock *BB, SPIRVEntry *Entry); SPIRVInstruction *transBinaryInst(BinaryOperator *B, SPIRVBasicBlock *BB); SPIRVInstruction *transCmpInst(CmpInst *Cmp, SPIRVBasicBlock *BB); SPIRVInstruction *transLifetimeIntrinsicInst(Op OC, IntrinsicInst *Intrinsic, SPIRVBasicBlock *BB); SPIRVValue *transAtomicStore(StoreInst *ST, SPIRVBasicBlock *BB); SPIRVValue *transAtomicLoad(LoadInst *LD, SPIRVBasicBlock *BB); void dumpUsers(Value *V); template bool oclGetExtInstIndex(const std::string &MangledName, const std::string &DemangledName, SPIRVWord *EntryPoint); void oclGetMutatedArgumentTypesByBuiltin(llvm::FunctionType *FT, std::map &ChangedType, Function *F); bool isBuiltinTransToInst(Function *F); bool isBuiltinTransToExtInst(Function *F, SPIRVExtInstSetKind *BuiltinSet = nullptr, SPIRVWord *EntryPoint = nullptr, SmallVectorImpl *Dec = nullptr); bool isKernel(Function *F); bool transMetadata(); bool transOCLMetadata(); SPIRVInstruction *transBuiltinToInst(StringRef DemangledName, CallInst *CI, SPIRVBasicBlock *BB); SPIRVValue *transBuiltinToConstant(StringRef DemangledName, CallInst *CI); SPIRVInstruction *transBuiltinToInstWithoutDecoration(Op OC, CallInst *CI, SPIRVBasicBlock *BB); void mutateFuncArgType(const std::map &ChangedType, Function *F); SPIRVValue *transSpcvCast(CallInst *CI, SPIRVBasicBlock *BB); SPIRVValue *oclTransSpvcCastSampler(CallInst *CI, SPIRVBasicBlock *BB); SPIRVValue *transUnaryInst(UnaryInstruction *U, SPIRVBasicBlock *BB); void transFunction(Function *I); SPIRV::SPIRVLinkageTypeKind transLinkageType(const GlobalValue *GV); bool isAnyFunctionReachableFromFunction( const Function *FS, const std::unordered_set Funcs) const; std::vector collectEntryPointInterfaces(SPIRVFunction *BF, Function *F); }; class LLVMToSPIRVPass : public PassInfoMixin { public: LLVMToSPIRVPass(SPIRVModule *SMod) : SMod(SMod) {} llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &MAM) { LLVMToSPIRVBase PassInstance(SMod); PassInstance.setOCLTypeToSPIRV(&MAM.getResult(M)); return PassInstance.runLLVMToSPIRV(M) ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all(); } private: SPIRVModule *SMod; }; class LLVMToSPIRVLegacy : public ModulePass, public LLVMToSPIRVBase { public: LLVMToSPIRVLegacy(SPIRVModule *SMod = nullptr) : ModulePass(ID), LLVMToSPIRVBase(SMod) {} virtual StringRef getPassName() const override { return "LLVMToSPIRV"; } bool runOnModule(Module &Mod) override { setOCLTypeToSPIRV(&getAnalysis()); return runLLVMToSPIRV(Mod); } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); } static char ID; }; } // namespace SPIRV #endif // SPIRVWRITER_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVWriterPass.cpp000066400000000000000000000036221506646577400226220ustar00rootroot00000000000000//===- SPIRVWriterPass.cpp - SPIRV writing pass -----------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // SPIRVWriterPass implementation. // //===----------------------------------------------------------------------===// #include "SPIRVWriterPass.h" #include "LLVMSPIRVLib.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" using namespace llvm; PreservedAnalyses SPIRVWriterPass::run(Module &M) { // FIXME: at the moment LLVM/SPIR-V translation errors are ignored. std::string Err; writeSpirv(&M, Opts, OS, Err); return PreservedAnalyses::all(); } namespace { class WriteSPIRVPass : public ModulePass { std::ostream &OS; // std::ostream to print on SPIRV::TranslatorOpts Opts; public: static char ID; // Pass identification, replacement for typeid WriteSPIRVPass(std::ostream &OS, const SPIRV::TranslatorOpts &Opts) : ModulePass(ID), OS(OS), Opts(Opts) {} StringRef getPassName() const override { return "SPIRV Writer"; } bool runOnModule(Module &M) override { // FIXME: at the moment LLVM/SPIR-V translation errors are ignored. std::string Err; writeSpirv(&M, Opts, OS, Err); return false; } }; } // namespace char WriteSPIRVPass::ID = 0; ModulePass *llvm::createSPIRVWriterPass(std::ostream &Str) { SPIRV::TranslatorOpts DefaultOpts; // To preserve old behavior of the translator, let's enable all extensions // by default in this API DefaultOpts.enableAllExtensions(); return createSPIRVWriterPass(Str, DefaultOpts); } ModulePass *llvm::createSPIRVWriterPass(std::ostream &Str, const SPIRV::TranslatorOpts &Opts) { return new WriteSPIRVPass(Str, Opts); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/SPIRVWriterPass.h000066400000000000000000000037301506646577400222670ustar00rootroot00000000000000//===------ SPIRVWriterPass.h - SPIRV writing pass --------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// \file /// /// This file provides a SPIRV writing pass. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_SPIRVWRITERPASS_H #define SPIRV_SPIRVWRITERPASS_H #include "LLVMSPIRVOpts.h" #include "llvm/ADT/StringRef.h" namespace llvm { class Module; class ModulePass; class PreservedAnalyses; /// \brief Create and return a pass that writes the module to the specified /// ostream. Note that this pass is designed for use with the legacy pass /// manager. ModulePass *createSPIRVWriterPass(std::ostream &Str); /// \brief Create and return a pass that writes the module to the specified /// ostream. Note that this pass is designed for use with the legacy pass /// manager. ModulePass *createSPIRVWriterPass(std::ostream &Str, const SPIRV::TranslatorOpts &Opts); /// \brief Pass for writing a module of IR out to a SPIRV file. /// /// Note that this is intended for use with the new pass manager. To construct /// a pass for the legacy pass manager, use the function above. class SPIRVWriterPass { std::ostream &OS; SPIRV::TranslatorOpts Opts; public: /// \brief Construct a SPIRV writer pass around a particular output stream. explicit SPIRVWriterPass(std::ostream &OS) : OS(OS) { Opts.enableAllExtensions(); } SPIRVWriterPass(std::ostream &OS, const SPIRV::TranslatorOpts &Opts) : OS(OS), Opts(Opts) {} /// \brief Run the SPIRV writer pass, and output the module to the selected /// output stream. PreservedAnalyses run(Module &M); static StringRef name() { return "SPIRVWriterPass"; } }; } // namespace llvm #endif // SPIRV_SPIRVWRITERPASS_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/VectorComputeUtil.cpp000077500000000000000000000140171506646577400233330ustar00rootroot00000000000000//=- VectorComputeUtil.cpp - vector compute utilities implemetation * C++ -*-=// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2020 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file implements translation of VC float control bits // //===----------------------------------------------------------------------===// #include "VectorComputeUtil.h" #include "SPIRVInternal.h" #include "llvm/IR/Metadata.h" using namespace VectorComputeUtil; using namespace SPIRV; enum VCFloatControl { VC_RTE = 0, // Round to nearest or even VC_RTP = 1 << 4, // Round towards +ve inf VC_RTN = 2 << 4, // Round towards -ve inf VC_RTZ = 3 << 4, // Round towards zero VC_DENORM_FTZ = 0, // Denorm mode flush to zero VC_DENORM_D_ALLOW = 1 << 6, // Denorm mode double allow VC_DENORM_F_ALLOW = 1 << 7, // Denorm mode float allow VC_DENORM_HF_ALLOW = 1 << 10, // Denorm mode half allow VC_FLOAT_MODE_IEEE = 0, // Single precision float IEEE mode VC_FLOAT_MODE_ALT = 1 // Single precision float ALT mode }; enum VCFloatControlMask { VC_ROUND_MASK = (VC_RTE | VC_RTP | VC_RTN | VC_RTZ), VC_FLOAT_MASK = (VC_FLOAT_MODE_IEEE | VC_FLOAT_MODE_ALT) }; typedef SPIRVMap FPRoundingModeControlBitMap; typedef SPIRVMap FPOperationModeControlBitMap; typedef SPIRVMap VCFloatTypeDenormMaskMap; template <> inline void SPIRVMap::init() { add(spv::FPRoundingModeRTE, VC_RTE); add(spv::FPRoundingModeRTP, VC_RTP); add(spv::FPRoundingModeRTN, VC_RTN); add(spv::FPRoundingModeRTZ, VC_RTZ); } template <> inline void SPIRVMap::init() { add(spv::FPOperationModeIEEE, VC_FLOAT_MODE_IEEE); add(spv::FPOperationModeALT, VC_FLOAT_MODE_ALT); } template <> inline void SPIRVMap::init() { add(Double, VC_DENORM_D_ALLOW); add(Float, VC_DENORM_F_ALLOW); add(Half, VC_DENORM_HF_ALLOW); } namespace VectorComputeUtil { FPRoundingMode getFPRoundingMode(unsigned FloatControl) noexcept { return FPRoundingModeControlBitMap::rmap( VCFloatControl(VC_ROUND_MASK & FloatControl)); } FPDenormMode getFPDenormMode(unsigned FloatControl, VCFloatType FloatType) noexcept { VCFloatControl DenormMask = VCFloatTypeDenormMaskMap::map(FloatType); // 1 Bit mask return (DenormMask == (DenormMask & FloatControl)) ? spv::FPDenormModePreserve : spv::FPDenormModeFlushToZero; } FPOperationMode getFPOperationMode(unsigned FloatControl) noexcept { return FPOperationModeControlBitMap::rmap( VCFloatControl(VC_FLOAT_MASK & FloatControl)); } unsigned getVCFloatControl(FPRoundingMode RoundMode) noexcept { return FPRoundingModeControlBitMap::map(RoundMode); } unsigned getVCFloatControl(FPOperationMode FloatMode) noexcept { return FPOperationModeControlBitMap::map(FloatMode); } unsigned getVCFloatControl(FPDenormMode DenormMode, VCFloatType FloatType) noexcept { if (DenormMode == spv::FPDenormModePreserve) return VCFloatTypeDenormMaskMap::map(FloatType); return VC_DENORM_FTZ; } SPIRVStorageClassKind getVCGlobalVarStorageClass(SPIRAddressSpace AddressSpace) noexcept { switch (AddressSpace) { case SPIRAS_Private: return StorageClassPrivate; case SPIRAS_Local: return StorageClassWorkgroup; case SPIRAS_Global: return StorageClassCrossWorkgroup; case SPIRAS_Constant: return StorageClassUniformConstant; default: assert(false && "Unexpected address space"); return StorageClassPrivate; } } SPIRAddressSpace getVCGlobalVarAddressSpace(SPIRVStorageClassKind StorageClass) noexcept { switch (StorageClass) { case StorageClassPrivate: return SPIRAS_Private; case StorageClassWorkgroup: return SPIRAS_Local; case StorageClassCrossWorkgroup: return SPIRAS_Global; case StorageClassUniformConstant: return SPIRAS_Constant; default: assert(false && "Unexpected storage class"); return SPIRAS_Private; } } std::string getVCBufferSurfaceName() { return std::string(kVCType::VCBufferSurface) + kAccessQualPostfix::Type; } std::string getVCBufferSurfaceName(SPIRVAccessQualifierKind Access) { return std::string(kVCType::VCBufferSurface) + getAccessQualifierPostfix(Access).str() + kAccessQualPostfix::Type; } } // namespace VectorComputeUtil SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/VectorComputeUtil.h000077500000000000000000000137561506646577400230110ustar00rootroot00000000000000//=- VectorComputeUtil.h - vector compute utilities declarations -*- C++ -*-=// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2020 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file declares translation of VectorComputeUtil float control bits, // and VC kernel metadata // //===----------------------------------------------------------------------===// #ifndef SPIRV_VCUTIL_H #define SPIRV_VCUTIL_H #include "SPIRVInternal.h" #include "SPIRVUtil.h" #include "spirv/unified1/spirv.hpp" namespace VectorComputeUtil { /////////////////////////////////////////////////////////////////////////////// // // Types // /////////////////////////////////////////////////////////////////////////////// enum VCFloatType { Double, Float, Half, }; FPRoundingMode getFPRoundingMode(unsigned FloatControl) noexcept; FPDenormMode getFPDenormMode(unsigned FloatControl, VCFloatType FloatType) noexcept; FPOperationMode getFPOperationMode(unsigned FloatControl) noexcept; unsigned getVCFloatControl(FPRoundingMode RoundMode) noexcept; unsigned getVCFloatControl(FPOperationMode FloatMode) noexcept; unsigned getVCFloatControl(FPDenormMode DenormMode, VCFloatType FloatType) noexcept; typedef SPIRV::SPIRVMap FPRoundingModeExecModeMap; typedef SPIRV::SPIRVMap FPOperationModeExecModeMap; typedef SPIRV::SPIRVMap FPDenormModeExecModeMap; typedef SPIRV::SPIRVMap VCFloatTypeSizeMap; /////////////////////////////////////////////////////////////////////////////// // // Functions // /////////////////////////////////////////////////////////////////////////////// SPIRVStorageClassKind getVCGlobalVarStorageClass(SPIRAddressSpace AddressSpace) noexcept; SPIRAddressSpace getVCGlobalVarAddressSpace(SPIRVStorageClassKind StorageClass) noexcept; std::string getVCBufferSurfaceName(); std::string getVCBufferSurfaceName(SPIRVAccessQualifierKind Access); } // namespace VectorComputeUtil /////////////////////////////////////////////////////////////////////////////// // // Constants // /////////////////////////////////////////////////////////////////////////////// namespace kVCMetadata { const static char VCFunction[] = "VCFunction"; const static char VCStackCall[] = "VCStackCall"; const static char VCArgumentIOKind[] = "VCArgumentIOKind"; const static char VCFloatControl[] = "VCFloatControl"; const static char VCSLMSize[] = "VCSLMSize"; const static char VCGlobalVariable[] = "VCGlobalVariable"; const static char VCVolatile[] = "VCVolatile"; const static char VCByteOffset[] = "VCByteOffset"; const static char VCSIMTCall[] = "VCSIMTCall"; const static char VCCallable[] = "VCCallable"; const static char VCSingleElementVector[] = "VCSingleElementVector"; const static char VCFCEntry[] = "VCFCEntry"; const static char VCMediaBlockIO[] = "VCMediaBlockIO"; const static char VCNamedBarrierCount[] = "VCNamedBarrierCount"; } // namespace kVCMetadata namespace kVCType { const static char VCBufferSurface[] = "intel.buffer"; } /////////////////////////////////////////////////////////////////////////////// // // Map definitions // /////////////////////////////////////////////////////////////////////////////// namespace SPIRV { template <> inline void SPIRVMap::init() { add(spv::FPRoundingModeRTE, spv::ExecutionModeRoundingModeRTE); add(spv::FPRoundingModeRTZ, spv::ExecutionModeRoundingModeRTZ); add(spv::FPRoundingModeRTP, spv::ExecutionModeRoundingModeRTPINTEL); add(spv::FPRoundingModeRTN, spv::ExecutionModeRoundingModeRTNINTEL); } template <> inline void SPIRVMap::init() { add(spv::FPDenormModeFlushToZero, spv::ExecutionModeDenormFlushToZero); add(spv::FPDenormModePreserve, spv::ExecutionModeDenormPreserve); } template <> inline void SPIRVMap::init() { add(spv::FPOperationModeIEEE, spv::ExecutionModeFloatingPointModeIEEEINTEL); add(spv::FPOperationModeALT, spv::ExecutionModeFloatingPointModeALTINTEL); } template <> inline void SPIRVMap::init() { add(VectorComputeUtil::Double, 64); add(VectorComputeUtil::Float, 32); add(VectorComputeUtil::Half, 16); } } // namespace SPIRV #endif // SPIRV_VCUTIL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/000077500000000000000000000000001506646577400205565ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/NonSemantic.AuxData.h000066400000000000000000000027701506646577400245010ustar00rootroot00000000000000/* ** Copyright (c) 2023 The Khronos Group Inc. ** ** Permission is hereby granted, free of charge, to any person obtaining a copy ** of this software and/or associated documentation files (the "Materials"), ** to deal in the Materials without restriction, including without limitation ** the rights to use, copy, modify, merge, publish, distribute, sublicense, ** and/or sell copies of the Materials, and to permit persons to whom the ** Materials are furnished to do so, subject to the following conditions: ** ** The above copyright notice and this permission notice shall be included in ** all copies or substantial portions of the Materials. ** ** MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS KHRONOS ** STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS SPECIFICATIONS AND ** HEADER INFORMATION ARE LOCATED AT https://www.khronos.org/registry/ ** ** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ** THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ** FROM,OUT OF OR IN CONNECTION WITH THE MATERIALS OR THE USE OR OTHER DEALINGS ** IN THE MATERIALS. */ namespace NonSemanticAuxData { enum Instruction { FunctionMetadata = 0, FunctionAttribute = 1, PreserveCount = 2 }; } // namespace NonSemanticAuxData SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/OpenCL.std.h000066400000000000000000000117541506646577400226500ustar00rootroot00000000000000// clang-format off /* ** Copyright (c) 2015 The Khronos Group Inc. ** ** Permission is hereby granted, free of charge, to any person obtaining a copy ** of this software and/or associated documentation files (the "Materials"), ** to deal in the Materials without restriction, including without limitation ** the rights to use, copy, modify, merge, publish, distribute, sublicense, ** and/or sell copies of the Materials, and to permit persons to whom the ** Materials are furnished to do so, subject to the following conditions: ** ** The above copyright notice and this permission notice shall be included in ** all copies or substantial portions of the Materials. ** ** MODIFICATIONS TO THIS FILE MAY MEAN IT NO LONGER ACCURATELY REFLECTS KHRONOS ** STANDARDS. THE UNMODIFIED, NORMATIVE VERSIONS OF KHRONOS SPECIFICATIONS AND ** HEADER INFORMATION ARE LOCATED AT https://www.khronos.org/registry/ ** ** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ** THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING ** FROM,OUT OF OR IN CONNECTION WITH THE MATERIALS OR THE USE OR OTHER DEALINGS ** IN THE MATERIALS. */ // // Author: Boaz Ouriel, Intel // namespace OpenCLLIB { enum Entrypoints { // math functions Acos = 0, Acosh = 1, Acospi = 2, Asin = 3, Asinh = 4, Asinpi = 5, Atan = 6, Atan2 = 7, Atanh = 8, Atanpi = 9, Atan2pi = 10, Cbrt = 11, Ceil = 12, Copysign = 13, Cos = 14, Cosh = 15, Cospi = 16, Erfc = 17, Erf = 18, Exp = 19, Exp2 = 20, Exp10 = 21, Expm1 = 22, Fabs = 23, Fdim = 24, Floor = 25, Fma = 26, Fmax = 27, Fmin = 28, Fmod = 29, Fract = 30, Frexp = 31, Hypot = 32, Ilogb = 33, Ldexp = 34, Lgamma = 35, Lgamma_r = 36, Log = 37, Log2 = 38, Log10 = 39, Log1p = 40, Logb = 41, Mad = 42, Maxmag = 43, Minmag = 44, Modf = 45, Nan = 46, Nextafter = 47, Pow = 48, Pown = 49, Powr = 50, Remainder = 51, Remquo = 52, Rint = 53, Rootn = 54, Round = 55, Rsqrt = 56, Sin = 57, Sincos = 58, Sinh = 59, Sinpi = 60, Sqrt = 61, Tan = 62, Tanh = 63, Tanpi = 64, Tgamma = 65, Trunc = 66, Half_cos = 67, Half_divide = 68, Half_exp = 69, Half_exp2 = 70, Half_exp10 = 71, Half_log = 72, Half_log2 = 73, Half_log10 = 74, Half_powr = 75, Half_recip = 76, Half_rsqrt = 77, Half_sin = 78, Half_sqrt = 79, Half_tan = 80, Native_cos = 81, Native_divide = 82, Native_exp = 83, Native_exp2 = 84, Native_exp10 = 85, Native_log = 86, Native_log2 = 87, Native_log10 = 88, Native_powr = 89, Native_recip = 90, Native_rsqrt = 91, Native_sin = 92, Native_sqrt = 93, Native_tan = 94, // Common FClamp = 95, Degrees = 96, FMax_common = 97, FMin_common = 98, Mix = 99, Radians = 100, Step = 101, Smoothstep = 102, Sign = 103, // Geometrics Cross = 104, Distance = 105, Length = 106, Normalize = 107, Fast_distance = 108, Fast_length = 109, Fast_normalize = 110, // Integers SAbs = 141, SAbs_diff = 142, SAdd_sat = 143, UAdd_sat = 144, SHadd = 145, UHadd = 146, SRhadd = 147, URhadd = 148, SClamp = 149, UClamp = 150, Clz = 151, Ctz = 152, SMad_hi = 153, UMad_sat = 154, SMad_sat = 155, SMax = 156, UMax = 157, SMin = 158, UMin = 159, SMul_hi = 160, Rotate = 161, SSub_sat = 162, USub_sat = 163, U_Upsample = 164, S_Upsample = 165, Popcount = 166, SMad24 = 167, UMad24 = 168, SMul24 = 169, UMul24 = 170, // Vector Loads/Stores Vloadn = 171, Vstoren = 172, Vload_half = 173, Vload_halfn = 174, Vstore_half = 175, Vstore_half_r = 176, Vstore_halfn = 177, Vstore_halfn_r = 178, Vloada_halfn = 179, Vstorea_halfn = 180, Vstorea_halfn_r = 181, // Vector Misc Shuffle = 182, Shuffle2 = 183, // Printf = 184, Prefetch = 185, // Relationals Bitselect = 186, Select = 187, // pipes Read_pipe = 188, Write_pipe = 189, Reserve_read_pipe = 190, Reserve_write_pipe = 191, Commit_read_pipe = 192, Commit_write_pipe = 193, Is_valid_reserve_id = 194, Work_group_reserve_read_pipe = 195, Work_group_reserve_write_pipe = 196, Work_group_commit_read_pipe = 197, Work_group_commit_write_pipe = 198, Get_pipe_num_packets = 199, Get_pipe_max_packets = 200, // more integers UAbs = 201, UAbs_diff = 202, UMul_hi = 203, UMad_hi = 204, }; } // end namespace OpenCL20 // clang-format on SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRV.debug.h000066400000000000000000001215401506646577400227220ustar00rootroot00000000000000#ifndef SPIRV_DEBUG_H #define SPIRV_DEBUG_H #include "SPIRVEnum.h" #include "SPIRVUtil.h" #include "spirv/unified1/spirv.hpp" #include "spirv_internal.hpp" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/DebugInfoMetadata.h" namespace SPIRVDebug { const unsigned int DebugInfoVersion = 0x00010000; static const std::string ProducerPrefix = {"Debug info producer: "}; static const std::string ChecksumKindPrefx = {"//__CSK_"}; // clang-format off // Need to update hasDbgInstParentScopeIdx each time we add new instruction // with ParentScopeIdx enum Instruction { DebugInfoNone = 0, CompilationUnit = 1, TypeBasic = 2, TypePointer = 3, TypeQualifier = 4, TypeArray = 5, TypeVector = 6, Typedef = 7, TypeFunction = 8, TypeEnum = 9, TypeComposite = 10, TypeMember = 11, TypeInheritance = 12, TypePtrToMember = 13, TypeTemplate = 14, TypeTemplateParameter = 15, TypeTemplateParameterPack = 16, TypeTemplateTemplateParameter = 17, GlobalVariable = 18, FunctionDeclaration = 19, Function = 20, LexicalBlock = 21, LexicalBlockDiscriminator = 22, Scope = 23, NoScope = 24, InlinedAt = 25, LocalVariable = 26, InlinedVariable = 27, Declare = 28, Value = 29, Operation = 30, Expression = 31, MacroDef = 32, MacroUndef = 33, ImportedEntity = 34, Source = 35, ModuleINTEL = 36, InstCount = 37, FunctionDefinition = 101, SourceContinued = 102, DebugLine = 103, DebugNoLine = 104, BuildIdentifier = 105, StoragePath = 106, EntryPoint = 107, Module = 200, TypeSubrange = 201, TypeArrayDynamic = 202, TypeString = 203 }; enum Flag { FlagIsProtected = 1 << 0, FlagIsPrivate = 1 << 1, FlagIsPublic = FlagIsPrivate | FlagIsProtected, FlagAccess = FlagIsPublic, FlagIsLocal = 1 << 2, FlagIsDefinition = 1 << 3, FlagIsFwdDecl = 1 << 4, FlagIsArtificial = 1 << 5, FlagIsExplicit = 1 << 6, FlagIsPrototyped = 1 << 7, FlagIsObjectPointer = 1 << 8, FlagIsStaticMember = 1 << 9, FlagIsIndirectVariable = 1 << 10, FlagIsLValueReference = 1 << 11, FlagIsRValueReference = 1 << 12, FlagIsOptimized = 1 << 13, FlagIsEnumClass = 1 << 14, FlagTypePassByValue = 1 << 15, FlagTypePassByReference = 1 << 16, FlagUnknownPhysicalLayout = 1 << 17, FlagBitField = 1 << 18 }; enum EncodingTag { Unspecified = 0, Address = 1, Boolean = 2, Float = 3, Signed = 4, SignedChar = 5, Unsigned = 6, UnsignedChar = 7, Complex = 8 }; enum CompositeTypeTag { Class = 0, Structure = 1, Union = 2 }; enum TypeQualifierTag { ConstType = 0, VolatileType = 1, RestrictType = 2, AtomicType = 3 }; enum ExpressionOpCode { Deref = 0, Plus = 1, Minus = 2, PlusUconst = 3, BitPiece = 4, Swap = 5, Xderef = 6, StackValue = 7, Constu = 8, Fragment = 9, Convert = 10, Addr = 11, Const1u = 12, Const1s = 13, Const2u = 14, Const2s = 15, Const4u = 16, Const4s = 17, Const8u = 18, Const8s = 19, Consts = 20, Dup = 21, Drop = 22, Over = 23, Pick = 24, Rot = 25, Abs = 26, And = 27, Div = 28, Mod = 29, Mul = 30, Neg = 31, Not = 32, Or = 33, Shl = 34, Shr = 35, Shra = 36, Xor = 37, Bra = 38, Eq = 39, Ge = 40, Gt = 41, Le = 42, Lt = 43, Ne = 44, Skip = 45, Lit0 = 46, Lit1 = 47, Lit2 = 48, Lit3 = 49, Lit4 = 50, Lit5 = 51, Lit6 = 52, Lit7 = 53, Lit8 = 54, Lit9 = 55, Lit10 = 56, Lit11 = 57, Lit12 = 58, Lit13 = 59, Lit14 = 60, Lit15 = 61, Lit16 = 62, Lit17 = 63, Lit18 = 64, Lit19 = 65, Lit20 = 66, Lit21 = 67, Lit22 = 68, Lit23 = 69, Lit24 = 70, Lit25 = 71, Lit26 = 72, Lit27 = 73, Lit28 = 74, Lit29 = 75, Lit30 = 76, Lit31 = 77, Reg0 = 78, Reg1 = 79, Reg2 = 80, Reg3 = 81, Reg4 = 82, Reg5 = 83, Reg6 = 84, Reg7 = 85, Reg8 = 86, Reg9 = 87, Reg10 = 88, Reg11 = 89, Reg12 = 90, Reg13 = 91, Reg14 = 92, Reg15 = 93, Reg16 = 94, Reg17 = 95, Reg18 = 96, Reg19 = 97, Reg20 = 98, Reg21 = 99, Reg22 = 100, Reg23 = 101, Reg24 = 102, Reg25 = 103, Reg26 = 104, Reg27 = 105, Reg28 = 106, Reg29 = 107, Reg30 = 108, Reg31 = 109, Breg0 = 110, Breg1 = 111, Breg2 = 112, Breg3 = 113, Breg4 = 114, Breg5 = 115, Breg6 = 116, Breg7 = 117, Breg8 = 118, Breg9 = 119, Breg10 = 120, Breg11 = 121, Breg12 = 122, Breg13 = 123, Breg14 = 124, Breg15 = 125, Breg16 = 126, Breg17 = 127, Breg18 = 128, Breg19 = 129, Breg20 = 130, Breg21 = 131, Breg22 = 132, Breg23 = 133, Breg24 = 134, Breg25 = 135, Breg26 = 136, Breg27 = 137, Breg28 = 138, Breg29 = 139, Breg30 = 140, Breg31 = 141, Regx = 142, Fbreg = 143, Bregx = 144, Piece = 145, DerefSize = 146, XderefSize = 147, Nop = 148, PushObjectAddress = 149, Call2 = 150, Call4 = 151, CallRef = 152, FormTlsAddress = 153, CallFrameCfa = 154, ImplicitValue = 155, ImplicitPointer = 156, Addrx = 157, Constx = 158, EntryValue = 159, ConstTypeOp = 160, RegvalType = 161, DerefType = 162, XderefType = 163, Reinterpret = 164, LLVMArg = 165, ImplicitPointerTag = 166, TagOffset = 167, }; enum ImportedEntityTag { ImportedModule = 0, ImportedDeclaration = 1, }; enum FileChecksumKind { MD5 = 0, SHA1 = 1, SHA256 = 2, }; namespace Operand { namespace CompilationUnit { enum { SPIRVDebugInfoVersionIdx = 0, DWARFVersionIdx = 1, SourceIdx = 2, LanguageIdx = 3, // For NonSemantic.Shader.DebugInfo.200 ProducerIdx = 4, MinOperandCount = 4 }; } namespace Source { enum { FileIdx = 0, TextIdx = 1, // For NonSemantic.Shader.DebugInfo.200 ChecksumKind = 1, ChecksumValue = 2, TextNonSemIdx = 3, MinOperandCount = 1, MaxOperandCount = 4 }; } namespace BuildIdentifier { enum { IdentifierIdx = 0, FlagsIdx = 1, OperandCount = 2 }; } namespace StoragePath { enum { PathIdx = 0, OperandCount = 1 }; } namespace TypeBasic { enum { NameIdx = 0, SizeIdx = 1, EncodingIdx = 2, // For NonSemantic Specs FlagsIdx = 3, OperandCountOCL = 3, OperandCountNonSemantic = 4 }; } namespace TypePointer { enum { BaseTypeIdx = 0, StorageClassIdx = 1, FlagsIdx = 2, OperandCount = 3 }; } namespace TypeQualifier { enum { BaseTypeIdx = 0, QualifierIdx = 1, OperandCount = 2 }; } namespace TypeArray { enum { BaseTypeIdx = 0, ComponentCountIdx = 1, SubrangesIdx = 1, MinOperandCount = 2 }; } namespace TypeArrayDynamic { enum { BaseTypeIdx = 0, DataLocationIdx = 1, AssociatedIdx = 2, AllocatedIdx = 3, RankIdx = 4, SubrangesIdx = 5, MinOperandCount = 6 }; } namespace TypeVector = TypeArray; namespace TypeSubrange { enum { LowerBoundIdx = 0, UpperBoundIdx = 1, CountIdx = 2, StrideIdx = 3, MinOperandCount = 3, MaxOperandCount = 4 }; } namespace TypeString { enum { NameIdx = 0, BaseTypeIdx = 1, DataLocationIdx = 2, SizeIdx = 3, LengthAddrIdx = 4, LengthSizeIdx = 5, MinOperandCount = 5 }; } namespace Typedef { enum { NameIdx = 0, BaseTypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, OperandCount = 6 }; } namespace TypeFunction { enum { FlagsIdx = 0, ReturnTypeIdx = 1, FirstParameterIdx = 2, MinOperandCount = 2 }; } namespace TypeEnum { enum { NameIdx = 0, UnderlyingTypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, SizeIdx = 6, FlagsIdx = 7, FirstEnumeratorIdx = 8, MinOperandCount = 8 }; } namespace TypeComposite { enum { NameIdx = 0, TagIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, LinkageNameIdx = 6, SizeIdx = 7, FlagsIdx = 8, FirstMemberIdx = 9, MinOperandCount = 9 }; } namespace TypeMember { namespace OpenCL { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, OffsetIdx = 6, SizeIdx = 7, FlagsIdx = 8, ValueIdx = 9, MinOperandCount = 9 }; } namespace NonSemantic { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, OffsetIdx = 5, SizeIdx = 6, FlagsIdx = 7, ValueIdx = 8, MinOperandCount = 8 }; } } // namespace TypeMember namespace TypeInheritance { namespace NonSemantic { enum { ParentIdx = 0, OffsetIdx = 1, SizeIdx = 2, FlagsIdx = 3, OperandCount = 4 }; } namespace OpenCL { enum { ChildIdx = 0, ParentIdx = 1, OffsetIdx = 2, SizeIdx = 3, FlagsIdx = 4, OperandCount = 5 }; } } // namespace TypeInheritance namespace TypePtrToMember { enum { MemberTypeIdx = 0, ParentIdx = 1, OperandCount = 2 }; } namespace TypeTemplate { enum { TargetIdx = 0, FirstParameterIdx = 1, MinOperandCount = 1 }; } namespace TypeTemplateParameter { enum { NameIdx = 0, TypeIdx = 1, ValueIdx = 2, SourceIdx = 3, LineIdx = 4, ColumnIdx = 5, OperandCount = 6 }; } namespace TypeTemplateTemplateParameter { enum { NameIdx = 0, TemplateNameIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, OperandCount = 5 }; } namespace TypeTemplateParameterPack { enum { NameIdx = 0, SourceIdx = 1, LineIdx = 2, ColumnIdx = 3, FirstParameterIdx = 4, MinOperandCount = 4 }; } namespace GlobalVariable { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, LinkageNameIdx = 6, VariableIdx = 7, FlagsIdx = 8, StaticMemberDeclarationIdx = 9, MinOperandCount = 9 }; } namespace FunctionDeclaration { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, LinkageNameIdx = 6, FlagsIdx = 7, OperandCount = 8 }; } namespace Function { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, LinkageNameIdx = 6, FlagsIdx = 7, ScopeLineIdx = 8, FunctionIdIdx = 9, DeclarationIdx = 10, MinOperandCount = 10, // Only for NonSemantic.Shader.DebugInfo.* // No operand FunctionId DeclarationNonSemIdx = 9, MinOperandCountNonSem = 9, // Only for NonSemantic.Shader.DebugInfo.200 TargetFunctionNameIdx = 10, }; } namespace FunctionDefinition { enum { FunctionIdx = 0, DefinitionIdx = 1, OperandCount = 2 }; } namespace SourceContinued { enum { TextIdx = 0, OperandCount = 1 }; } namespace DebugLine { enum { SourceIdx = 0, StartIdx = 1, EndIdx = 2, ColumnStartIdx = 3, ColumnEndIdx = 4, OperandCount = 5 }; } namespace DebugNoLine { enum { OperandCount = 0 }; } namespace EntryPoint { enum { EntryPointIdx = 0, CompilationUnitIdx = 1, CompilerSignatureIdx = 2, CommandLineArgsIdx = 3, OperandCount = 4 }; } namespace LexicalBlock { enum { SourceIdx = 0, LineIdx = 1, ColumnIdx = 2, ParentIdx = 3, MinOperandCount = 4, NameIdx = 4, InlineNamespaceIdx = 5, }; } namespace LexicalBlockDiscriminator { enum { SourceIdx = 0, DiscriminatorIdx = 1, ParentIdx = 2, OperandCount = 3 }; } namespace Scope { enum { ScopeIdx = 0, InlinedAtIdx = 1, MinOperandCount = 1 }; } namespace NoScope { // No operands } namespace InlinedAt { namespace OpenCL { enum { LineIdx = 0, ScopeIdx = 1, InlinedIdx = 2, MinOperandCount = 2 }; } // namespace OpenCL namespace NonSemantic { enum { LineIdx = 0, ColumnIdx = 1, ScopeIdx = 2, InlinedIdx = 3, MinOperandCount = 3 }; } // namespace NonSemantic } // namespace ImportedEntity namespace LocalVariable { enum { NameIdx = 0, TypeIdx = 1, SourceIdx = 2, LineIdx = 3, ColumnIdx = 4, ParentIdx = 5, FlagsIdx = 6, ArgNumberIdx = 7, MinOperandCount = 7 }; } namespace InlinedVariable { enum { VariableIdx = 0, InlinedIdx = 1, OperandCount = 2 }; } namespace DebugDeclare { enum { DebugLocalVarIdx = 0, VariableIdx = 1, ExpressionIdx = 2, OperandCount = 3 }; } namespace DebugValue { enum { DebugLocalVarIdx = 0, ValueIdx = 1, ExpressionIdx = 2, FirstIndexOperandIdx = 3, MinOperandCount = 3 }; } namespace Operation { enum { OpCodeIdx = 0 }; static std::map OpCountMap { { Deref, 1 }, { Plus, 1 }, { Minus, 1 }, { PlusUconst, 2 }, { BitPiece, 3 }, { Swap, 1 }, { Xderef, 1 }, { StackValue, 1 }, { Constu, 2 }, { Fragment, 3 }, { Convert, 3 }, { Addr, 2 }, { Const1u, 2 }, { Const1s, 2 }, { Const2u, 2 }, { Const2s, 2 }, { Const4u, 2 }, { Const4s, 2 }, { Const8u, 2 }, { Const8s, 2 }, { Consts, 2 }, { Dup, 1 }, { Drop, 1 }, { Over, 1 }, { Pick, 1 }, { Rot, 1 }, { Abs, 1 }, { And, 1 }, { Div, 1 }, { Mod, 1 }, { Mul, 1 }, { Neg, 1 }, { Not, 1 }, { Or, 1 }, { Shl, 1 }, { Shr, 1 }, { Shra, 1 }, { Xor, 1 }, { Bra, 2 }, { Eq, 1 }, { Ge, 1 }, { Gt, 1 }, { Le, 1 }, { Lt, 1 }, { Ne, 1 }, { Skip, 2 }, { Lit0, 1 }, { Lit1, 1 }, { Lit2, 1 }, { Lit3, 1 }, { Lit4, 1 }, { Lit5, 1 }, { Lit6, 1 }, { Lit7, 1 }, { Lit8, 1 }, { Lit9, 1 }, { Lit10, 1 }, { Lit11, 1 }, { Lit12, 1 }, { Lit13, 1 }, { Lit14, 1 }, { Lit15, 1 }, { Lit16, 1 }, { Lit17, 1 }, { Lit18, 1 }, { Lit19, 1 }, { Lit20, 1 }, { Lit21, 1 }, { Lit22, 1 }, { Lit23, 1 }, { Lit24, 1 }, { Lit25, 1 }, { Lit26, 1 }, { Lit27, 1 }, { Lit28, 1 }, { Lit29, 1 }, { Lit30, 1 }, { Lit31, 1 }, { Reg0, 1 }, { Reg1, 1 }, { Reg2, 1 }, { Reg3, 1 }, { Reg4, 1 }, { Reg5, 1 }, { Reg6, 1 }, { Reg7, 1 }, { Reg8, 1 }, { Reg9, 1 }, { Reg10, 1 }, { Reg11, 1 }, { Reg12, 1 }, { Reg13, 1 }, { Reg14, 1 }, { Reg15, 1 }, { Reg16, 1 }, { Reg17, 1 }, { Reg18, 1 }, { Reg19, 1 }, { Reg20, 1 }, { Reg21, 1 }, { Reg22, 1 }, { Reg23, 1 }, { Reg24, 1 }, { Reg25, 1 }, { Reg26, 1 }, { Reg27, 1 }, { Reg28, 1 }, { Reg29, 1 }, { Reg30, 1 }, { Reg31, 1 }, { Breg0, 2 }, { Breg1, 2 }, { Breg2, 2 }, { Breg3, 2 }, { Breg4, 2 }, { Breg5, 2 }, { Breg6, 2 }, { Breg7, 2 }, { Breg8, 2 }, { Breg9, 2 }, { Breg10, 2 }, { Breg11, 2 }, { Breg12, 2 }, { Breg13, 2 }, { Breg14, 2 }, { Breg15, 2 }, { Breg16, 2 }, { Breg17, 2 }, { Breg18, 2 }, { Breg19, 2 }, { Breg20, 2 }, { Breg21, 2 }, { Breg22, 2 }, { Breg23, 2 }, { Breg24, 2 }, { Breg25, 2 }, { Breg26, 2 }, { Breg27, 2 }, { Breg28, 2 }, { Breg29, 2 }, { Breg30, 2 }, { Breg31, 2 }, { Regx, 2 }, { Fbreg, 1 }, { Bregx, 3 }, { Piece, 2 }, { DerefSize, 2 }, { XderefSize, 2 }, { Nop, 1 }, { PushObjectAddress, 1 }, { Call2, 2 }, { Call4, 2 }, { CallRef, 2 }, { FormTlsAddress, 1 }, { CallFrameCfa, 1 }, { ImplicitValue, 3 }, { ImplicitPointer, 3 }, { Addrx, 2 }, { Constx, 2 }, { EntryValue, 3 }, { ConstTypeOp, 4 }, { RegvalType, 3 }, { DerefType, 3 }, { XderefType, 3 }, { Reinterpret, 2 }, { LLVMArg, 2 }, { ImplicitPointerTag, 2 }, { TagOffset, 2 }, }; } namespace ImportedEntity { namespace OpenCL { // it's bugged version, note 2nd index is missing // FIXME: need to remove it after some graceful period enum { NameIdx = 0, TagIdx = 1, SourceIdx = 3, EntityIdx = 4, LineIdx = 5, ColumnIdx = 6, ParentIdx = 7, OperandCount = 8 }; } // namespace OpenCL namespace NonSemantic { enum { NameIdx = 0, TagIdx = 1, SourceIdx = 2, EntityIdx = 3, LineIdx = 4, ColumnIdx = 5, ParentIdx = 6, OperandCount = 7 }; } // namespace NonSemantic } // namespace ImportedEntity namespace ModuleINTEL { enum { NameIdx = 0, SourceIdx = 1, LineIdx = 2, ParentIdx = 3, ConfigMacrosIdx = 4, IncludePathIdx = 5, ApiNotesIdx = 6, IsDeclIdx = 7, OperandCount = 8 }; } // helper function to get parent scope of debug instruction, to be used // to determine with which compile unit the particular instruction relates inline bool hasDbgInstParentScopeIdx( const uint32_t Kind, uint32_t &ParentScopeIdx, const SPIRV::SPIRVExtInstSetKind ExtKind = SPIRV::SPIRVEIS_OpenCL) { switch (Kind) { case SPIRVDebug::Typedef: ParentScopeIdx = Typedef::ParentIdx; return true; case SPIRVDebug::TypeEnum: ParentScopeIdx = TypeEnum::ParentIdx; return true; case SPIRVDebug::TypeComposite: ParentScopeIdx = TypeMember::OpenCL::ParentIdx; return true; case SPIRVDebug::TypeInheritance: if (ExtKind == SPIRV::SPIRVEIS_OpenCL_DebugInfo_100) ParentScopeIdx = TypeInheritance::OpenCL::ParentIdx; else ParentScopeIdx = TypeInheritance::NonSemantic::ParentIdx; return true; case SPIRVDebug::TypePtrToMember: ParentScopeIdx = TypePtrToMember::ParentIdx; return true; case SPIRVDebug::Function: ParentScopeIdx = Function::ParentIdx; return true; case SPIRVDebug::EntryPoint: ParentScopeIdx = EntryPoint::CompilationUnitIdx; return true; case SPIRVDebug::LexicalBlock: ParentScopeIdx = LexicalBlock::ParentIdx; return true; case SPIRVDebug::LexicalBlockDiscriminator: ParentScopeIdx = LexicalBlockDiscriminator::ParentIdx; return true; case SPIRVDebug::Scope: ParentScopeIdx = Scope::ScopeIdx; return true; case SPIRVDebug::InlinedAt: if (ExtKind == SPIRV::SPIRVEIS_NonSemantic_Shader_DebugInfo_200) ParentScopeIdx = InlinedAt::NonSemantic::ScopeIdx; else ParentScopeIdx = InlinedAt::OpenCL::ScopeIdx; return true; case SPIRVDebug::LocalVariable: ParentScopeIdx = LocalVariable::ParentIdx; return true; case SPIRVDebug::ImportedEntity: if (ExtKind == SPIRV::SPIRVEIS_OpenCL_DebugInfo_100) ParentScopeIdx = ImportedEntity::OpenCL::ParentIdx; else ParentScopeIdx = ImportedEntity::NonSemantic::ParentIdx; return true; case SPIRVDebug::ModuleINTEL: ParentScopeIdx = ModuleINTEL::ParentIdx; return true; case SPIRVDebug::Module: ParentScopeIdx = ModuleINTEL::ParentIdx; return true; default: return false; } } } // namespace Operand } // namespace SPIRVDebug using namespace llvm; inline spv::SourceLanguage convertDWARFSourceLangToSPIRV(dwarf::SourceLanguage DwarfLang) { switch (DwarfLang) { // When updating this function, make sure to also // update convertSPIRVSourceLangToDWARF() case dwarf::SourceLanguage::DW_LANG_C_plus_plus_14: case dwarf::SourceLanguage::DW_LANG_C_plus_plus: return spv::SourceLanguage::SourceLanguageCPP_for_OpenCL; case dwarf::SourceLanguage::DW_LANG_C99: case dwarf::SourceLanguage::DW_LANG_OpenCL: return spv::SourceLanguage::SourceLanguageOpenCL_C; default: return spv::SourceLanguage::SourceLanguageUnknown; } } inline bool isSPIRVSourceLangValid(unsigned SourceLang) { switch (SourceLang) { // When updating this function, make sure to also // update convertSPIRVSourceLangToDWARF() case spv::SourceLanguage::SourceLanguageOpenCL_CPP: case spv::SourceLanguage::SourceLanguageCPP_for_OpenCL: case spv::SourceLanguage::SourceLanguageOpenCL_C: case spv::SourceLanguage::SourceLanguageESSL: case spv::SourceLanguage::SourceLanguageGLSL: case spv::SourceLanguage::SourceLanguageHLSL: case spv::SourceLanguage::SourceLanguageUnknown: return true; default: return false; } } inline dwarf::SourceLanguage convertSPIRVSourceLangToDWARF(unsigned SourceLang) { switch (SourceLang) { // When updating this function, make sure to also // update convertDWARFSourceLangToSPIRV() case spv::SourceLanguage::SourceLanguageOpenCL_CPP: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_14; case spv::SourceLanguage::SourceLanguageCPP_for_OpenCL: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_14; case spv::SourceLanguage::SourceLanguageOpenCL_C: case spv::SourceLanguage::SourceLanguageESSL: case spv::SourceLanguage::SourceLanguageGLSL: case spv::SourceLanguage::SourceLanguageHLSL: case spv::SourceLanguage::SourceLanguageUnknown: default: return dwarf::DW_LANG_OpenCL; } } inline spv::SourceLanguage convertDWARFSourceLangToSPIRVNonSemanticDbgInfo( dwarf::SourceLanguage DwarfLang) { switch (DwarfLang) { // When updating this function, make sure to also // update convertSPIRVSourceLangToDWARFNonSemanticDbgInfo() case dwarf::SourceLanguage::DW_LANG_OpenCL: return spv::SourceLanguage::SourceLanguageOpenCL_C; case dwarf::SourceLanguage::DW_LANG_C_plus_plus_14: return spv::internal::SourceLanguageCPP14; case dwarf::SourceLanguage::DW_LANG_C_plus_plus_11: return spv::internal::SourceLanguageCPP11; case dwarf::SourceLanguage::DW_LANG_C_plus_plus_03: return spv::internal::SourceLanguageCPP03; case dwarf::SourceLanguage::DW_LANG_C_plus_plus: return spv::internal::SourceLanguageCPP; case dwarf::SourceLanguage::DW_LANG_C: return spv::internal::SourceLanguageC; case dwarf::SourceLanguage::DW_LANG_C99: return spv::internal::SourceLanguageC99; case dwarf::SourceLanguage::DW_LANG_C11: return spv::internal::SourceLanguageC11; case dwarf::SourceLanguage::DW_LANG_Python: return spv::internal::SourceLanguagePython; case dwarf::SourceLanguage::DW_LANG_Julia: return spv::internal::SourceLanguageJulia; case dwarf::SourceLanguage::DW_LANG_Rust: return spv::internal::SourceLanguageRust; case dwarf::SourceLanguage::DW_LANG_D: return spv::internal::SourceLanguageD; case dwarf::SourceLanguage::DW_LANG_Fortran77: return spv::internal::SourceLanguageFortran77; case dwarf::SourceLanguage::DW_LANG_Fortran90: return spv::internal::SourceLanguageFortran90; case dwarf::SourceLanguage::DW_LANG_Fortran95: return spv::internal::SourceLanguageFortran95; case dwarf::SourceLanguage::DW_LANG_Fortran03: return spv::internal::SourceLanguageFortran2003; case dwarf::SourceLanguage::DW_LANG_Fortran08: return spv::internal::SourceLanguageFortran2008; default: return spv::SourceLanguage::SourceLanguageUnknown; } } inline dwarf::SourceLanguage convertSPIRVSourceLangToDWARFNonSemanticDbgInfo(unsigned SourceLang) { switch (SourceLang) { // When updating this function, make sure to also // update convertDWARFSourceLangToSPIRVNonSemanticDbgInfo() case spv::SourceLanguage::SourceLanguageOpenCL_CPP: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_14; case spv::SourceLanguage::SourceLanguageCPP_for_OpenCL: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_14; case spv::internal::SourceLanguageCPP14: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_14; case spv::internal::SourceLanguageCPP11: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_11; case spv::internal::SourceLanguageCPP03: return dwarf::SourceLanguage::DW_LANG_C_plus_plus_03; case spv::internal::SourceLanguageCPP: return dwarf::SourceLanguage::DW_LANG_C_plus_plus; case spv::internal::SourceLanguageC: return dwarf::SourceLanguage::DW_LANG_C; case spv::internal::SourceLanguageC99: return dwarf::SourceLanguage::DW_LANG_C99; case spv::internal::SourceLanguageC11: return dwarf::SourceLanguage::DW_LANG_C11; case spv::internal::SourceLanguagePython: return dwarf::SourceLanguage::DW_LANG_Python; case spv::internal::SourceLanguageJulia: return dwarf::SourceLanguage::DW_LANG_Julia; case spv::internal::SourceLanguageRust: return dwarf::SourceLanguage::DW_LANG_Rust; case spv::internal::SourceLanguageD: return dwarf::SourceLanguage::DW_LANG_D; case spv::internal::SourceLanguageFortran77: return dwarf::SourceLanguage::DW_LANG_Fortran77; case spv::internal::SourceLanguageFortran90: return dwarf::SourceLanguage::DW_LANG_Fortran90; case spv::internal::SourceLanguageFortran95: return dwarf::SourceLanguage::DW_LANG_Fortran95; case spv::internal::SourceLanguageFortran2003: return dwarf::SourceLanguage::DW_LANG_Fortran03; case spv::internal::SourceLanguageFortran2008: return dwarf::SourceLanguage::DW_LANG_Fortran08; case spv::SourceLanguage::SourceLanguageOpenCL_C: case spv::SourceLanguage::SourceLanguageESSL: case spv::SourceLanguage::SourceLanguageGLSL: case spv::SourceLanguage::SourceLanguageHLSL: case spv::SourceLanguage::SourceLanguageUnknown: default: return dwarf::DW_LANG_OpenCL; } } namespace SPIRV { typedef SPIRVMap DbgEncodingMap; template <> inline void DbgEncodingMap::init() { add(static_cast(0), SPIRVDebug::Unspecified); add(dwarf::DW_ATE_address, SPIRVDebug::Address); add(dwarf::DW_ATE_boolean, SPIRVDebug::Boolean); add(dwarf::DW_ATE_float, SPIRVDebug::Float); add(dwarf::DW_ATE_signed, SPIRVDebug::Signed); add(dwarf::DW_ATE_signed_char, SPIRVDebug::SignedChar); add(dwarf::DW_ATE_unsigned, SPIRVDebug::Unsigned); add(dwarf::DW_ATE_unsigned_char, SPIRVDebug::UnsignedChar); add(dwarf::DW_ATE_complex_float, SPIRVDebug::Complex); } typedef SPIRVMap DbgTypeQulifierMap; template <> inline void DbgTypeQulifierMap::init() { add(dwarf::DW_TAG_const_type, SPIRVDebug::ConstType); add(dwarf::DW_TAG_volatile_type, SPIRVDebug::VolatileType); add(dwarf::DW_TAG_restrict_type, SPIRVDebug::RestrictType); add(dwarf::DW_TAG_atomic_type, SPIRVDebug::AtomicType); } typedef SPIRVMap DbgCompositeTypeMap; template <> inline void DbgCompositeTypeMap::init() { add(dwarf::DW_TAG_class_type, SPIRVDebug::Class); add(dwarf::DW_TAG_structure_type, SPIRVDebug::Structure); add(dwarf::DW_TAG_union_type, SPIRVDebug::Union); } typedef SPIRVMap DbgExpressionOpCodeMap; template <> inline void DbgExpressionOpCodeMap::init() { add(dwarf::DW_OP_deref, SPIRVDebug::Deref); add(dwarf::DW_OP_plus, SPIRVDebug::Plus); add(dwarf::DW_OP_minus, SPIRVDebug::Minus); add(dwarf::DW_OP_plus_uconst, SPIRVDebug::PlusUconst); add(dwarf::DW_OP_bit_piece, SPIRVDebug::BitPiece); add(dwarf::DW_OP_swap, SPIRVDebug::Swap); add(dwarf::DW_OP_xderef, SPIRVDebug::Xderef); add(dwarf::DW_OP_stack_value, SPIRVDebug::StackValue); add(dwarf::DW_OP_constu, SPIRVDebug::Constu); add(dwarf::DW_OP_LLVM_fragment, SPIRVDebug::Fragment); add(dwarf::DW_OP_LLVM_convert, SPIRVDebug::Convert); add(dwarf::DW_OP_addr, SPIRVDebug::Addr); add(dwarf::DW_OP_const1u, SPIRVDebug::Const1u); add(dwarf::DW_OP_const1s, SPIRVDebug::Const1s); add(dwarf::DW_OP_const2u, SPIRVDebug::Const2u); add(dwarf::DW_OP_const2s, SPIRVDebug::Const2s); add(dwarf::DW_OP_const4u, SPIRVDebug::Const4u); add(dwarf::DW_OP_const4s, SPIRVDebug::Const4s); add(dwarf::DW_OP_const8u, SPIRVDebug::Const8u); add(dwarf::DW_OP_const8s, SPIRVDebug::Const8s); add(dwarf::DW_OP_consts, SPIRVDebug::Consts); add(dwarf::DW_OP_dup, SPIRVDebug::Dup); add(dwarf::DW_OP_drop, SPIRVDebug::Drop); add(dwarf::DW_OP_over, SPIRVDebug::Over); add(dwarf::DW_OP_pick, SPIRVDebug::Pick); add(dwarf::DW_OP_rot, SPIRVDebug::Rot); add(dwarf::DW_OP_abs, SPIRVDebug::Abs); add(dwarf::DW_OP_and, SPIRVDebug::And); add(dwarf::DW_OP_div, SPIRVDebug::Div); add(dwarf::DW_OP_mod, SPIRVDebug::Mod); add(dwarf::DW_OP_mul, SPIRVDebug::Mul); add(dwarf::DW_OP_neg, SPIRVDebug::Neg); add(dwarf::DW_OP_not, SPIRVDebug::Not); add(dwarf::DW_OP_or, SPIRVDebug::Or); add(dwarf::DW_OP_shl, SPIRVDebug::Shl); add(dwarf::DW_OP_shr, SPIRVDebug::Shr); add(dwarf::DW_OP_shra, SPIRVDebug::Shra); add(dwarf::DW_OP_xor, SPIRVDebug::Xor); add(dwarf::DW_OP_bra, SPIRVDebug::Bra); add(dwarf::DW_OP_eq, SPIRVDebug::Eq); add(dwarf::DW_OP_ge, SPIRVDebug::Ge); add(dwarf::DW_OP_gt, SPIRVDebug::Gt); add(dwarf::DW_OP_le, SPIRVDebug::Le); add(dwarf::DW_OP_lt, SPIRVDebug::Lt); add(dwarf::DW_OP_ne, SPIRVDebug::Ne); add(dwarf::DW_OP_skip, SPIRVDebug::Skip); add(dwarf::DW_OP_lit0, SPIRVDebug::Lit0); add(dwarf::DW_OP_lit1, SPIRVDebug::Lit1); add(dwarf::DW_OP_lit2, SPIRVDebug::Lit2); add(dwarf::DW_OP_lit3, SPIRVDebug::Lit3); add(dwarf::DW_OP_lit4, SPIRVDebug::Lit4); add(dwarf::DW_OP_lit5, SPIRVDebug::Lit5); add(dwarf::DW_OP_lit6, SPIRVDebug::Lit6); add(dwarf::DW_OP_lit7, SPIRVDebug::Lit7); add(dwarf::DW_OP_lit8, SPIRVDebug::Lit8); add(dwarf::DW_OP_lit9, SPIRVDebug::Lit9); add(dwarf::DW_OP_lit10, SPIRVDebug::Lit10); add(dwarf::DW_OP_lit11, SPIRVDebug::Lit11); add(dwarf::DW_OP_lit12, SPIRVDebug::Lit12); add(dwarf::DW_OP_lit13, SPIRVDebug::Lit13); add(dwarf::DW_OP_lit14, SPIRVDebug::Lit14); add(dwarf::DW_OP_lit15, SPIRVDebug::Lit15); add(dwarf::DW_OP_lit16, SPIRVDebug::Lit16); add(dwarf::DW_OP_lit17, SPIRVDebug::Lit17); add(dwarf::DW_OP_lit18, SPIRVDebug::Lit18); add(dwarf::DW_OP_lit19, SPIRVDebug::Lit19); add(dwarf::DW_OP_lit20, SPIRVDebug::Lit20); add(dwarf::DW_OP_lit21, SPIRVDebug::Lit21); add(dwarf::DW_OP_lit22, SPIRVDebug::Lit22); add(dwarf::DW_OP_lit23, SPIRVDebug::Lit23); add(dwarf::DW_OP_lit24, SPIRVDebug::Lit24); add(dwarf::DW_OP_lit25, SPIRVDebug::Lit25); add(dwarf::DW_OP_lit26, SPIRVDebug::Lit26); add(dwarf::DW_OP_lit27, SPIRVDebug::Lit27); add(dwarf::DW_OP_lit28, SPIRVDebug::Lit28); add(dwarf::DW_OP_lit29, SPIRVDebug::Lit29); add(dwarf::DW_OP_lit30, SPIRVDebug::Lit30); add(dwarf::DW_OP_lit31, SPIRVDebug::Lit31); add(dwarf::DW_OP_reg0, SPIRVDebug::Reg0); add(dwarf::DW_OP_reg1, SPIRVDebug::Reg1); add(dwarf::DW_OP_reg2, SPIRVDebug::Reg2); add(dwarf::DW_OP_reg3, SPIRVDebug::Reg3); add(dwarf::DW_OP_reg4, SPIRVDebug::Reg4); add(dwarf::DW_OP_reg5, SPIRVDebug::Reg5); add(dwarf::DW_OP_reg6, SPIRVDebug::Reg6); add(dwarf::DW_OP_reg7, SPIRVDebug::Reg7); add(dwarf::DW_OP_reg8, SPIRVDebug::Reg8); add(dwarf::DW_OP_reg9, SPIRVDebug::Reg9); add(dwarf::DW_OP_reg10, SPIRVDebug::Reg10); add(dwarf::DW_OP_reg11, SPIRVDebug::Reg11); add(dwarf::DW_OP_reg12, SPIRVDebug::Reg12); add(dwarf::DW_OP_reg13, SPIRVDebug::Reg13); add(dwarf::DW_OP_reg14, SPIRVDebug::Reg14); add(dwarf::DW_OP_reg15, SPIRVDebug::Reg15); add(dwarf::DW_OP_reg16, SPIRVDebug::Reg16); add(dwarf::DW_OP_reg17, SPIRVDebug::Reg17); add(dwarf::DW_OP_reg18, SPIRVDebug::Reg18); add(dwarf::DW_OP_reg19, SPIRVDebug::Reg19); add(dwarf::DW_OP_reg20, SPIRVDebug::Reg20); add(dwarf::DW_OP_reg21, SPIRVDebug::Reg21); add(dwarf::DW_OP_reg22, SPIRVDebug::Reg22); add(dwarf::DW_OP_reg23, SPIRVDebug::Reg23); add(dwarf::DW_OP_reg24, SPIRVDebug::Reg24); add(dwarf::DW_OP_reg25, SPIRVDebug::Reg25); add(dwarf::DW_OP_reg26, SPIRVDebug::Reg26); add(dwarf::DW_OP_reg27, SPIRVDebug::Reg27); add(dwarf::DW_OP_reg28, SPIRVDebug::Reg28); add(dwarf::DW_OP_reg29, SPIRVDebug::Reg29); add(dwarf::DW_OP_reg30, SPIRVDebug::Reg30); add(dwarf::DW_OP_reg31, SPIRVDebug::Reg31); add(dwarf::DW_OP_breg0, SPIRVDebug::Breg0); add(dwarf::DW_OP_breg1, SPIRVDebug::Breg1); add(dwarf::DW_OP_breg2, SPIRVDebug::Breg2); add(dwarf::DW_OP_breg3, SPIRVDebug::Breg3); add(dwarf::DW_OP_breg4, SPIRVDebug::Breg4); add(dwarf::DW_OP_breg5, SPIRVDebug::Breg5); add(dwarf::DW_OP_breg6, SPIRVDebug::Breg6); add(dwarf::DW_OP_breg7, SPIRVDebug::Breg7); add(dwarf::DW_OP_breg8, SPIRVDebug::Breg8); add(dwarf::DW_OP_breg9, SPIRVDebug::Breg9); add(dwarf::DW_OP_breg10, SPIRVDebug::Breg10); add(dwarf::DW_OP_breg11, SPIRVDebug::Breg11); add(dwarf::DW_OP_breg12, SPIRVDebug::Breg12); add(dwarf::DW_OP_breg13, SPIRVDebug::Breg13); add(dwarf::DW_OP_breg14, SPIRVDebug::Breg14); add(dwarf::DW_OP_breg15, SPIRVDebug::Breg15); add(dwarf::DW_OP_breg16, SPIRVDebug::Breg16); add(dwarf::DW_OP_breg17, SPIRVDebug::Breg17); add(dwarf::DW_OP_breg18, SPIRVDebug::Breg18); add(dwarf::DW_OP_breg19, SPIRVDebug::Breg19); add(dwarf::DW_OP_breg20, SPIRVDebug::Breg20); add(dwarf::DW_OP_breg21, SPIRVDebug::Breg21); add(dwarf::DW_OP_breg22, SPIRVDebug::Breg22); add(dwarf::DW_OP_breg23, SPIRVDebug::Breg23); add(dwarf::DW_OP_breg24, SPIRVDebug::Breg24); add(dwarf::DW_OP_breg25, SPIRVDebug::Breg25); add(dwarf::DW_OP_breg26, SPIRVDebug::Breg26); add(dwarf::DW_OP_breg27, SPIRVDebug::Breg27); add(dwarf::DW_OP_breg28, SPIRVDebug::Breg28); add(dwarf::DW_OP_breg29, SPIRVDebug::Breg29); add(dwarf::DW_OP_breg30, SPIRVDebug::Breg30); add(dwarf::DW_OP_breg31, SPIRVDebug::Breg31); add(dwarf::DW_OP_regx, SPIRVDebug::Regx); add(dwarf::DW_OP_bregx, SPIRVDebug::Bregx); add(dwarf::DW_OP_piece, SPIRVDebug::Piece); add(dwarf::DW_OP_deref_size, SPIRVDebug::DerefSize ); add(dwarf::DW_OP_xderef_size, SPIRVDebug::XderefSize ); add(dwarf::DW_OP_nop, SPIRVDebug::Nop); add(dwarf::DW_OP_push_object_address, SPIRVDebug::PushObjectAddress ); add(dwarf::DW_OP_call2, SPIRVDebug::Call2); add(dwarf::DW_OP_call4, SPIRVDebug::Call4); add(dwarf::DW_OP_call_ref, SPIRVDebug::CallRef); add(dwarf::DW_OP_form_tls_address, SPIRVDebug::FormTlsAddress); add(dwarf::DW_OP_call_frame_cfa, SPIRVDebug::CallFrameCfa); add(dwarf::DW_OP_implicit_value, SPIRVDebug::ImplicitValue); add(dwarf::DW_OP_implicit_pointer, SPIRVDebug::ImplicitPointer); add(dwarf::DW_OP_addrx, SPIRVDebug::Addrx); add(dwarf::DW_OP_constx, SPIRVDebug::Constx); add(dwarf::DW_OP_entry_value, SPIRVDebug::EntryValue); add(dwarf::DW_OP_const_type, SPIRVDebug::ConstTypeOp); add(dwarf::DW_OP_regval_type, SPIRVDebug::RegvalType); add(dwarf::DW_OP_deref_type, SPIRVDebug::DerefType); add(dwarf::DW_OP_xderef_type, SPIRVDebug::XderefType); add(dwarf::DW_OP_reinterpret, SPIRVDebug::Reinterpret); add(dwarf::DW_OP_LLVM_arg, SPIRVDebug::LLVMArg); add(dwarf::DW_OP_LLVM_implicit_pointer, SPIRVDebug::ImplicitPointerTag); add(dwarf::DW_OP_LLVM_tag_offset, SPIRVDebug::TagOffset); } typedef SPIRVMap DbgImportedEntityMap; template <> inline void DbgImportedEntityMap::init() { add(dwarf::DW_TAG_imported_module, SPIRVDebug::ImportedModule); add(dwarf::DW_TAG_imported_declaration, SPIRVDebug::ImportedDeclaration); } typedef SPIRVMap DbgChecksumKindMap; template <> inline void DbgChecksumKindMap::init() { add(llvm::DIFile::CSK_MD5, SPIRVDebug::MD5); add(llvm::DIFile::CSK_SHA1, SPIRVDebug::SHA1); add(llvm::DIFile::CSK_SHA256, SPIRVDebug::SHA256); } } // namespace SPIRV #endif // SPIRV_DEBUG_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVAsm.h000066400000000000000000000112101506646577400222660ustar00rootroot00000000000000//===- SPIRVAsm.h - --*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the inline assembler entries defined in SPIRV spec with op /// codes. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVASM_H #define SPIRV_LIBSPIRV_SPIRVASM_H #include "SPIRVEntry.h" #include "SPIRVInstruction.h" #include "SPIRVValue.h" namespace SPIRV { class SPIRVAsmTargetINTEL : public SPIRVEntry { public: static const SPIRVWord FixedWC = 2; static const Op OC = OpAsmTargetINTEL; // Complete constructor SPIRVAsmTargetINTEL(SPIRVModule *M, SPIRVId TheId, const std::string &TheTarget) : SPIRVEntry(M, FixedWC + getSizeInWords(TheTarget), OC, TheId), Target(TheTarget) { validate(); } // Incomplete constructor SPIRVAsmTargetINTEL() : SPIRVEntry(OC) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityAsmINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_inline_assembly; } const std::string &getTarget() const { return Target; } protected: void validate() const override { SPIRVEntry::validate(); assert(WordCount > FixedWC); assert(OpCode == OC); } _SPIRV_DEF_ENCDEC2(Id, Target) std::string Target; }; class SPIRVAsmINTEL : public SPIRVValue { public: static const SPIRVWord FixedWC = 5; static const Op OC = OpAsmINTEL; // Complete constructor SPIRVAsmINTEL(SPIRVModule *M, SPIRVTypeFunction *TheFunctionType, SPIRVId TheId, SPIRVAsmTargetINTEL *TheTarget, const std::string &TheInstructions, const std::string &TheConstraints) : SPIRVValue(M, FixedWC + getSizeInWords(TheInstructions) + getSizeInWords(TheConstraints), OC, TheFunctionType->getReturnType(), TheId), Target(TheTarget), FunctionType(TheFunctionType), Instructions(TheInstructions), Constraints(TheConstraints) { validate(); } // Incomplete constructor SPIRVAsmINTEL() : SPIRVValue(OC) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityAsmINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_inline_assembly; } const std::string &getInstructions() const { return Instructions; } const std::string &getConstraints() const { return Constraints; } SPIRVTypeFunction *getFunctionType() const { return FunctionType; } protected: _SPIRV_DEF_ENCDEC6(Type, Id, FunctionType, Target, Instructions, Constraints) void validate() const override { SPIRVValue::validate(); assert(WordCount > FixedWC); assert(OpCode == OC); } SPIRVAsmTargetINTEL *Target; SPIRVTypeFunction *FunctionType; std::string Instructions; std::string Constraints; }; class SPIRVAsmCallINTEL : public SPIRVInstruction { public: static const SPIRVWord FixedWC = 4; static const Op OC = OpAsmCallINTEL; // Complete constructor SPIRVAsmCallINTEL(SPIRVId TheId, SPIRVAsmINTEL *TheAsm, const std::vector &TheArgs, SPIRVBasicBlock *TheBB) : SPIRVInstruction(FixedWC + TheArgs.size(), OC, TheAsm->getType(), TheId, TheBB), Asm(TheAsm), Args(TheArgs) { validate(); } // Incomplete constructor SPIRVAsmCallINTEL() : SPIRVInstruction(OC) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityAsmINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_inline_assembly; } bool isOperandLiteral(unsigned int Index) const override { return false; } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Args.resize(TheWordCount - FixedWC); } const std::vector &getArguments() const { return Args; } SPIRVAsmINTEL *getAsm() const { return Asm; } protected: _SPIRV_DEF_ENCDEC4(Type, Id, Asm, Args) void validate() const override { SPIRVInstruction::validate(); assert(WordCount >= FixedWC); assert(OpCode == OC); assert(getBasicBlock() && "Invalid BB"); assert(getBasicBlock()->getModule() == Asm->getModule()); } SPIRVAsmINTEL *Asm; std::vector Args; }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVASM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVBasicBlock.cpp000066400000000000000000000071161506646577400241070ustar00rootroot00000000000000//===- SPIRVBasicBlock.cpp - SPIR-V Basic Block -----------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements SPIRV basic block. /// //===----------------------------------------------------------------------===// #include "SPIRVBasicBlock.h" #include "SPIRVEntry.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVStream.h" #include "SPIRVValue.h" #include using namespace SPIRV; SPIRVBasicBlock::SPIRVBasicBlock(SPIRVId TheId, SPIRVFunction *Func) : SPIRVValue(Func->getModule(), 2, OpLabel, TheId), ParentF(Func) { setAttr(); validate(); } SPIRVDecoder SPIRVBasicBlock::getDecoder(std::istream &IS) { return SPIRVDecoder(IS, *this); } /// Assume I contains valid Id. SPIRVInstruction * SPIRVBasicBlock::addInstruction(SPIRVInstruction *I, const SPIRVInstruction *InsertBefore) { assert(I && "Invalid instruction"); Module->add(I); I->setParent(this); if (InsertBefore) { auto Pos = find(InsertBefore); // If insertion of a new instruction before the one passed to the function // is illegal, insertion before the returned instruction is guaranteed // to retain correct instruction order in a block if (Pos != InstVec.begin() && (isa(*std::prev(Pos)) || isa(*std::prev(Pos)))) --Pos; InstVec.insert(Pos, I); } else InstVec.push_back(I); return I; } void SPIRVBasicBlock::encodeChildren(spv_ostream &O) const { O << SPIRVNL(); for (size_t I = 0, E = InstVec.size(); I != E; ++I) O << *InstVec[I]; } _SPIRV_IMP_ENCDEC1(SPIRVBasicBlock, Id) void SPIRVBasicBlock::setScope(SPIRVEntry *Scope) { assert(Scope && Scope->getOpCode() == OpFunction && "Invalid scope"); setParent(static_cast(Scope)); } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVBasicBlock.h000066400000000000000000000104711506646577400235520ustar00rootroot00000000000000//===- SPIRVBasicBlock.h - SPIR-V Basic Block -------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Basic Block class for SPIR-V. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVBASICBLOCK_H #define SPIRV_LIBSPIRV_SPIRVBASICBLOCK_H #include "SPIRVValue.h" #include namespace SPIRV { class SPIRVFunction; class SPIRVInstruction; class SPIRVDecoder; class SPIRVBasicBlock : public SPIRVValue { public: SPIRVBasicBlock(SPIRVId TheId, SPIRVFunction *Func); SPIRVBasicBlock() : SPIRVValue(OpLabel), ParentF(NULL) { setAttr(); } SPIRVDecoder getDecoder(std::istream &IS) override; SPIRVFunction *getParent() const { return ParentF; } size_t getNumInst() const { return InstVec.size(); } SPIRVInstruction *getInst(size_t I) const { return InstVec[I]; } SPIRVInstruction *getPrevious(const SPIRVInstruction *I) const { auto Loc = find(I); if (Loc == InstVec.end() || Loc == InstVec.begin()) return nullptr; return *(--Loc); } SPIRVInstruction *getNext(const SPIRVInstruction *I) const { auto Loc = find(I); if (Loc == InstVec.end()) return nullptr; ++Loc; if (Loc == InstVec.end()) return nullptr; return *Loc; } // Return the last instruction in the BB or nullptr if the BB is empty. const SPIRVInstruction *getTerminateInstr() const { return InstVec.empty() ? nullptr : InstVec.back(); } void setScope(SPIRVEntry *Scope) override; void setParent(SPIRVFunction *F) { ParentF = F; } SPIRVInstruction * addInstruction(SPIRVInstruction *I, const SPIRVInstruction *InsertBefore = nullptr); void eraseInstruction(const SPIRVInstruction *I) { auto Loc = find(I); assert(Loc != InstVec.end()); InstVec.erase(Loc); } void setAttr() { setHasNoType(); } _SPIRV_DCL_ENCDEC void encodeChildren(spv_ostream &) const override; void validate() const override { SPIRVValue::validate(); assert(ParentF && "Invalid parent function"); } private: SPIRVFunction *ParentF; typedef std::vector SPIRVInstructionVector; SPIRVInstructionVector InstVec; SPIRVInstructionVector::const_iterator find(const SPIRVInstruction *Inst) const { return std::find(InstVec.begin(), InstVec.end(), Inst); } SPIRVInstructionVector::iterator find(const SPIRVInstruction *Inst) { return std::find(InstVec.begin(), InstVec.end(), Inst); } }; typedef SPIRVBasicBlock SPIRVLabel; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVBASICBLOCK_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVDebug.cpp000066400000000000000000000060251506646577400231370ustar00rootroot00000000000000//===- SPIRVDebug.cpp - SPIR-V Debug Utility --------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines variables for enabling/disabling SPIR-V debug macro. /// //===----------------------------------------------------------------------===// #include "SPIRVDebug.h" #include "llvm/IR/Verifier.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "spirv-regularization" using namespace SPIRV; bool SPIRV::SPIRVDbgEnable = false; SPIRV::SPIRVDbgErrorHandlingKinds SPIRV::SPIRVDbgError = SPIRVDbgErrorHandlingKinds::Exit; bool SPIRV::SPIRVDbgErrorMsgIncludesSourceInfo = true; namespace SPIRV { llvm::cl::opt VerifyRegularizationPasses( "spirv-verify-regularize-passes", llvm::cl::init(_SPIRVDBG), llvm::cl::desc( "Verify module after each pass in LLVM regularization phase")); void verifyRegularizationPass(llvm::Module &M, const std::string &PassName) { if (VerifyRegularizationPasses) { std::string Err; llvm::raw_string_ostream ErrorOS(Err); if (llvm::verifyModule(M, &ErrorOS)) { LLVM_DEBUG(llvm::errs() << "Failed to verify module after pass: " << PassName << "\n" << ErrorOS.str()); } } } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVDebug.h000066400000000000000000000072311506646577400226040ustar00rootroot00000000000000//===- SPIRVDebug.h - SPIR-V Debug Utility ----------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Macros and variables for debugging SPIRV. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVDEBUG_H #define SPIRV_LIBSPIRV_SPIRVDEBUG_H #include "SPIRVUtil.h" #include #include namespace llvm { class Module; } namespace SPIRV { // Include source file and line number in error message. extern bool SPIRVDbgErrorMsgIncludesSourceInfo; // Enable assert or exit on error enum class SPIRVDbgErrorHandlingKinds { Abort, Exit, Ignore }; extern SPIRVDbgErrorHandlingKinds SPIRVDbgError; // Enable debug output. extern bool SPIRVDbgEnable; void verifyRegularizationPass(llvm::Module &, const std::string &); #ifndef _SPIRVDBG #if !defined(NDEBUG) || defined(_DEBUG) #define _SPIRVDBG true #else #define _SPIRVDBG false #endif #endif #if _SPIRVDBG #define SPIRVDBG(x) \ if (SPIRVDbgEnable) { \ x; \ } // Output stream for SPIRV debug information. inline spv_ostream &spvdbgs() { return std::cerr; } #else #define SPIRVDBG(x) // Minimal std::basic_ostream mock that ignores everything being printed via // operator<< class dev_null_stream { public: void flush() {} }; template const dev_null_stream &operator<<(const dev_null_stream &Out, const T &) { return Out; } template const dev_null_stream &&operator<<(const dev_null_stream &&Out, const T &) { return std::move(Out); } // Output stream for SPIRV debug information. inline dev_null_stream &spvdbgs() { static dev_null_stream Out; return Out; } #endif } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVDEBUG_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVDecorate.cpp000066400000000000000000000212551506646577400236410ustar00rootroot00000000000000//===- SPIRVDecorate.cpp -SPIR-V Decorations --------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements SPIR-V decorations. /// //===----------------------------------------------------------------------===// #include "SPIRVDecorate.h" #include "SPIRVModule.h" #include "SPIRVStream.h" #include "SPIRVValue.h" namespace SPIRV { template spv_ostream &operator<<(spv_ostream &O, const std::vector &V) { for (auto &I : V) O << *I; return O; } SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget) : SPIRVAnnotationGeneric(TheTarget->getModule(), WC, OC, TheTarget->getId()), Dec(TheDec), Owner(nullptr) { validate(); updateModuleVersion(); } SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V) : SPIRVAnnotationGeneric(TheTarget->getModule(), WC, OC, TheTarget->getId()), Dec(TheDec), Owner(nullptr) { Literals.push_back(V); validate(); updateModuleVersion(); } SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V1, SPIRVWord V2) : SPIRVDecorateGeneric(OC, WC, TheDec, TheTarget, V1) { Literals.push_back(V2); validate(); updateModuleVersion(); } SPIRVDecorateGeneric::SPIRVDecorateGeneric(Op OC) : SPIRVAnnotationGeneric(OC), Dec(DecorationRelaxedPrecision), Owner(nullptr) {} Decoration SPIRVDecorateGeneric::getDecorateKind() const { return Dec; } SPIRVWord SPIRVDecorateGeneric::getLiteral(size_t I) const { assert(I <= Literals.size() && "Out of bounds"); return Literals[I]; } std::vector SPIRVDecorateGeneric::getVecLiteral() const { return Literals; } size_t SPIRVDecorateGeneric::getLiteralCount() const { return Literals.size(); } void SPIRVDecorate::encode(spv_ostream &O) const { SPIRVEncoder Encoder = getEncoder(O); Encoder << Target << Dec; switch (static_cast(Dec)) { case DecorationLinkageAttributes: SPIRVDecorateLinkageAttr::encodeLiterals(Encoder, Literals); break; case DecorationMemoryINTEL: SPIRVDecorateMemoryINTELAttr::encodeLiterals(Encoder, Literals); break; case DecorationMergeINTEL: SPIRVDecorateMergeINTELAttr::encodeLiterals(Encoder, Literals); break; case DecorationUserSemantic: SPIRVDecorateUserSemanticAttr::encodeLiterals(Encoder, Literals); break; case internal::DecorationHostAccessINTEL: SPIRVDecorateHostAccessINTELLegacy::encodeLiterals(Encoder, Literals); break; case DecorationHostAccessINTEL: SPIRVDecorateHostAccessINTEL::encodeLiterals(Encoder, Literals); break; case DecorationInitModeINTEL: SPIRVDecorateInitModeINTEL::encodeLiterals(Encoder, Literals); break; default: Encoder << Literals; } } void SPIRVDecorate::setWordCount(SPIRVWord Count) { WordCount = Count; Literals.resize(WordCount - FixedWC); } void SPIRVDecorate::decode(std::istream &I) { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Target >> Dec; switch (static_cast(Dec)) { case DecorationLinkageAttributes: SPIRVDecorateLinkageAttr::decodeLiterals(Decoder, Literals); break; case DecorationMemoryINTEL: SPIRVDecorateMemoryINTELAttr::decodeLiterals(Decoder, Literals); break; case DecorationMergeINTEL: SPIRVDecorateMergeINTELAttr::decodeLiterals(Decoder, Literals); break; case DecorationUserSemantic: SPIRVDecorateUserSemanticAttr::decodeLiterals(Decoder, Literals); break; case internal::DecorationHostAccessINTEL: SPIRVDecorateHostAccessINTELLegacy::decodeLiterals(Decoder, Literals); break; case DecorationHostAccessINTEL: SPIRVDecorateHostAccessINTEL::decodeLiterals(Decoder, Literals); break; default: Decoder >> Literals; } getOrCreateTarget()->addDecorate(this); } void SPIRVDecorateId::encode(spv_ostream &O) const { SPIRVEncoder Encoder = getEncoder(O); Encoder << Target << Dec << Literals; } void SPIRVDecorateId::setWordCount(SPIRVWord Count) { WordCount = Count; Literals.resize(WordCount - FixedWC); } void SPIRVDecorateId::decode(std::istream &I) { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Target >> Dec >> Literals; getOrCreateTarget()->addDecorate(this); } void SPIRVMemberDecorate::encode(spv_ostream &O) const { SPIRVEncoder Encoder = getEncoder(O); Encoder << Target << MemberNumber << Dec; switch (Dec) { case DecorationMemoryINTEL: SPIRVDecorateMemoryINTELAttr::encodeLiterals(Encoder, Literals); break; case DecorationMergeINTEL: SPIRVDecorateMergeINTELAttr::encodeLiterals(Encoder, Literals); break; case DecorationUserSemantic: SPIRVDecorateUserSemanticAttr::encodeLiterals(Encoder, Literals); break; default: Encoder << Literals; } } void SPIRVMemberDecorate::setWordCount(SPIRVWord Count) { WordCount = Count; Literals.resize(WordCount - FixedWC); } void SPIRVMemberDecorate::decode(std::istream &I) { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Target >> MemberNumber >> Dec; switch (Dec) { case DecorationMemoryINTEL: SPIRVDecorateMemoryINTELAttr::decodeLiterals(Decoder, Literals); break; case DecorationMergeINTEL: SPIRVDecorateMergeINTELAttr::decodeLiterals(Decoder, Literals); break; case DecorationUserSemantic: SPIRVDecorateUserSemanticAttr::decodeLiterals(Decoder, Literals); break; default: Decoder >> Literals; } getOrCreateTarget()->addMemberDecorate(this); } void SPIRVDecorationGroup::encode(spv_ostream &O) const { getEncoder(O) << Id; } void SPIRVDecorationGroup::decode(std::istream &I) { getDecoder(I) >> Id; Module->addDecorationGroup(this); } void SPIRVDecorationGroup::encodeAll(spv_ostream &O) const { O << Decorations; SPIRVEntry::encodeAll(O); } void SPIRVGroupDecorateGeneric::encode(spv_ostream &O) const { getEncoder(O) << DecorationGroup << Targets; } void SPIRVGroupDecorateGeneric::decode(std::istream &I) { getDecoder(I) >> DecorationGroup >> Targets; Module->addGroupDecorateGeneric(this); } void SPIRVGroupDecorate::decorateTargets() { for (auto &I : Targets) { auto Target = getOrCreate(I); for (auto &Dec : DecorationGroup->getDecorations()) { assert(Dec->isDecorate()); Target->addDecorate(static_cast(Dec)); } } } void SPIRVGroupMemberDecorate::decorateTargets() { for (auto &I : Targets) { auto Target = getOrCreate(I); for (auto &Dec : DecorationGroup->getDecorations()) { assert(Dec->isMemberDecorate()); Target->addMemberDecorate(static_cast(Dec)); } } } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVDecorate.h000066400000000000000000001033041506646577400233020ustar00rootroot00000000000000//===- SPIRVDecorate.h - SPIR-V Decorations ---------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V decorations. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVDECORATE_H #define SPIRV_LIBSPIRV_SPIRVDECORATE_H #include "SPIRVEntry.h" #include "SPIRVStream.h" #include "SPIRVUtil.h" #include #include #include namespace SPIRV { class SPIRVDecorationGroup; class SPIRVDecorateGeneric : public SPIRVAnnotationGeneric { public: // Complete constructor for decorations without literals SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget); // Complete constructor for decorations with one word literal SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V); // Complete constructor for decorations with two word literals SPIRVDecorateGeneric(Op OC, SPIRVWord WC, Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V1, SPIRVWord V2); // Incomplete constructor SPIRVDecorateGeneric(Op OC); SPIRVWord getLiteral(size_t) const; std::vector getVecLiteral() const; Decoration getDecorateKind() const; size_t getLiteralCount() const; SPIRVDecorationGroup *getOwner() const { return Owner; } void setOwner(SPIRVDecorationGroup *Owner) { this->Owner = Owner; } SPIRVCapVec getRequiredCapability() const override { switch (Dec) { case DecorationBuiltIn: { // Return the BuiltIn's capabilities. BuiltIn BI = static_cast(Literals.back()); return getCapability(BI); } case DecorationUniform: if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) return getVec(CapabilityUniformDecoration); return getVec(CapabilityShader); default: return getCapability(Dec); } } SPIRVWord getRequiredSPIRVVersion() const override { switch (Dec) { case DecorationSpecId: if (getModule()->hasCapability(CapabilityKernel)) return static_cast(VersionNumber::SPIRV_1_1); else return static_cast(VersionNumber::SPIRV_1_0); case DecorationMaxByteOffset: return static_cast(VersionNumber::SPIRV_1_1); case DecorationUserSemantic: case DecorationCounterBuffer: return static_cast(VersionNumber::SPIRV_1_4); default: return static_cast(VersionNumber::SPIRV_1_0); } } protected: Decoration Dec; std::vector Literals; SPIRVDecorationGroup *Owner; // Owning decorate group }; typedef std::vector SPIRVDecorateVec; class SPIRVDecorate : public SPIRVDecorateGeneric { public: static const Op OC = OpDecorate; static const SPIRVWord FixedWC = 3; // Complete constructor for decorations without literals SPIRVDecorate(Decoration TheDec, SPIRVEntry *TheTarget) : SPIRVDecorateGeneric(OC, 3, TheDec, TheTarget) {} // Complete constructor for decorations with one word literal SPIRVDecorate(Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V) : SPIRVDecorateGeneric(OC, 4, TheDec, TheTarget, V) {} // Complete constructor for decorations with two word literals SPIRVDecorate(Decoration TheDec, SPIRVEntry *TheTarget, SPIRVWord V1, SPIRVWord V2) : SPIRVDecorateGeneric(OC, 5, TheDec, TheTarget, V1, V2) {} // Incomplete constructor SPIRVDecorate() : SPIRVDecorateGeneric(OC) {} llvm::Optional getRequiredExtension() const override { switch (static_cast(Dec)) { case DecorationRegisterINTEL: case DecorationMemoryINTEL: case DecorationNumbanksINTEL: case DecorationBankwidthINTEL: case DecorationMaxPrivateCopiesINTEL: case DecorationSinglepumpINTEL: case DecorationDoublepumpINTEL: case DecorationMaxReplicatesINTEL: case DecorationSimpleDualPortINTEL: case DecorationMergeINTEL: case DecorationBankBitsINTEL: case DecorationForcePow2DepthINTEL: return ExtensionID::SPV_INTEL_fpga_memory_attributes; case DecorationBurstCoalesceINTEL: case DecorationCacheSizeINTEL: case DecorationDontStaticallyCoalesceINTEL: case DecorationPrefetchINTEL: return ExtensionID::SPV_INTEL_fpga_memory_accesses; case DecorationReferencedIndirectlyINTEL: case internal::DecorationArgumentAttributeINTEL: return ExtensionID::SPV_INTEL_function_pointers; case DecorationIOPipeStorageINTEL: return ExtensionID::SPV_INTEL_io_pipes; case DecorationBufferLocationINTEL: return ExtensionID::SPV_INTEL_fpga_buffer_location; case DecorationFunctionFloatingPointModeINTEL: case DecorationFunctionRoundingModeINTEL: case DecorationFunctionDenormModeINTEL: return ExtensionID::SPV_INTEL_float_controls2; case DecorationStallEnableINTEL: return ExtensionID::SPV_INTEL_fpga_cluster_attributes; case DecorationFuseLoopsInFunctionINTEL: return ExtensionID::SPV_INTEL_loop_fuse; case internal::DecorationMathOpDSPModeINTEL: return ExtensionID::SPV_INTEL_fpga_dsp_control; case internal::DecorationInitiationIntervalINTEL: return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes; case internal::DecorationMaxConcurrencyINTEL: return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes; case internal::DecorationPipelineEnableINTEL: return ExtensionID::SPV_INTEL_fpga_invocation_pipelining_attributes; case internal::DecorationRuntimeAlignedINTEL: return ExtensionID::SPV_INTEL_runtime_aligned; case internal::DecorationHostAccessINTEL: case internal::DecorationInitModeINTEL: case internal::DecorationImplementInCSRINTEL: return ExtensionID::SPV_INTEL_global_variable_decorations; case DecorationInitModeINTEL: case DecorationImplementInRegisterMapINTEL: return ExtensionID::SPV_INTEL_global_variable_fpga_decorations; case DecorationHostAccessINTEL: return ExtensionID::SPV_INTEL_global_variable_host_access; case DecorationFPMaxErrorDecorationINTEL: return ExtensionID::SPV_INTEL_fp_max_error; case DecorationCacheControlLoadINTEL: case DecorationCacheControlStoreINTEL: return ExtensionID::SPV_INTEL_cache_controls; default: return {}; } } _SPIRV_DCL_ENCDEC void setWordCount(SPIRVWord) override; void validate() const override { SPIRVDecorateGeneric::validate(); assert(WordCount == Literals.size() + FixedWC); } }; class SPIRVDecorateString : public SPIRVDecorate {}; class SPIRVDecorateId : public SPIRVDecorateGeneric { public: static const Op OC = OpDecorateId; static const SPIRVWord FixedWC = 3; // Complete constructor for decorations with one id operand SPIRVDecorateId(Decoration TheDec, SPIRVEntry *TheTarget, SPIRVId V) : SPIRVDecorateGeneric(OC, 4, TheDec, TheTarget, V) {} // Incomplete constructor SPIRVDecorateId() : SPIRVDecorateGeneric(OC) {} llvm::Optional getRequiredExtension() const override { switch (static_cast(Dec)) { case DecorationAliasScopeINTEL: case DecorationNoAliasINTEL: return ExtensionID::SPV_INTEL_memory_access_aliasing; default: return {}; } } _SPIRV_DCL_ENCDEC void setWordCount(SPIRVWord) override; void validate() const override { SPIRVDecorateGeneric::validate(); assert(WordCount == Literals.size() + FixedWC); } }; class SPIRVDecorateLinkageAttr : public SPIRVDecorate { public: // Complete constructor for LinkageAttributes decorations SPIRVDecorateLinkageAttr(SPIRVEntry *TheTarget, const std::string &Name, SPIRVLinkageTypeKind Kind) : SPIRVDecorate(DecorationLinkageAttributes, TheTarget) { for (auto &I : getVec(Name)) Literals.push_back(I); Literals.push_back(Kind); WordCount += Literals.size(); } // Incomplete constructor SPIRVDecorateLinkageAttr() : SPIRVDecorate() {} std::string getLinkageName() const { return getString(Literals.cbegin(), Literals.cend() - 1); } SPIRVLinkageTypeKind getLinkageType() const { return (SPIRVLinkageTypeKind)Literals.back(); } static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << getString(Literals.cbegin(), Literals.cend() - 1); Encoder << (SPIRVLinkageTypeKind)Literals.back(); } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { std::string Name; Decoder >> Name; SPIRVLinkageTypeKind Kind; Decoder >> Kind; std::copy_n(getVec(Name).begin(), Literals.size() - 1, Literals.begin()); Literals.back() = Kind; } else #endif Decoder >> Literals; } llvm::Optional getRequiredExtension() const override { if (getLinkageType() == SPIRVLinkageTypeKind::LinkageTypeLinkOnceODR) return ExtensionID::SPV_KHR_linkonce_odr; return {}; } }; class SPIRVMemberDecorate : public SPIRVDecorateGeneric { public: static const Op OC = OpMemberDecorate; static const SPIRVWord FixedWC = 4; // Complete constructor for decorations without literals SPIRVMemberDecorate(Decoration TheDec, SPIRVWord Member, SPIRVEntry *TheTarget) : SPIRVDecorateGeneric(OC, 4, TheDec, TheTarget), MemberNumber(Member) {} // Complete constructor for decorations with one word literal SPIRVMemberDecorate(Decoration TheDec, SPIRVWord Member, SPIRVEntry *TheTarget, SPIRVWord V) : SPIRVDecorateGeneric(OC, 5, TheDec, TheTarget, V), MemberNumber(Member) {} // Incomplete constructor SPIRVMemberDecorate() : SPIRVDecorateGeneric(OC), MemberNumber(SPIRVWORD_MAX) {} llvm::Optional getRequiredExtension() const override { switch (static_cast(Dec)) { case DecorationRegisterINTEL: case DecorationMemoryINTEL: case DecorationNumbanksINTEL: case DecorationBankwidthINTEL: case DecorationMaxPrivateCopiesINTEL: case DecorationSinglepumpINTEL: case DecorationDoublepumpINTEL: case DecorationMaxReplicatesINTEL: case DecorationSimpleDualPortINTEL: case DecorationMergeINTEL: case DecorationBankBitsINTEL: case DecorationForcePow2DepthINTEL: return ExtensionID::SPV_INTEL_fpga_memory_attributes; case DecorationBurstCoalesceINTEL: case DecorationCacheSizeINTEL: case DecorationDontStaticallyCoalesceINTEL: case DecorationPrefetchINTEL: return ExtensionID::SPV_INTEL_fpga_memory_accesses; case DecorationIOPipeStorageINTEL: return ExtensionID::SPV_INTEL_io_pipes; case DecorationBufferLocationINTEL: return ExtensionID::SPV_INTEL_fpga_buffer_location; case internal::DecorationRuntimeAlignedINTEL: return ExtensionID::SPV_INTEL_runtime_aligned; default: return {}; } } SPIRVWord getMemberNumber() const { return MemberNumber; } std::pair getPair() const { return std::make_pair(MemberNumber, Dec); } _SPIRV_DCL_ENCDEC void setWordCount(SPIRVWord) override; void validate() const override { SPIRVDecorateGeneric::validate(); assert(WordCount == Literals.size() + FixedWC); } protected: SPIRVWord MemberNumber; }; class SPIRVMemberDecorateString : public SPIRVMemberDecorate {}; class SPIRVDecorationGroup : public SPIRVEntry { public: static const Op OC = OpDecorationGroup; static const SPIRVWord WC = 2; // Complete constructor. Does not populate Decorations. SPIRVDecorationGroup(SPIRVModule *TheModule, SPIRVId TheId) : SPIRVEntry(TheModule, WC, OC, TheId) { validate(); }; // Incomplete constructor SPIRVDecorationGroup() : SPIRVEntry(OC) {} void encodeAll(spv_ostream &O) const override; _SPIRV_DCL_ENCDEC // Move the given decorates to the decoration group void takeDecorates(SPIRVDecorateVec &Decs) { Decorations = std::move(Decs); for (auto &I : Decorations) const_cast(I)->setOwner(this); Decs.clear(); } SPIRVDecorateVec &getDecorations() { return Decorations; } protected: SPIRVDecorateVec Decorations; void validate() const override { assert(OpCode == OC); assert(WordCount == WC); } }; class SPIRVGroupDecorateGeneric : public SPIRVEntryNoIdGeneric { public: static const SPIRVWord FixedWC = 2; // Complete constructor SPIRVGroupDecorateGeneric(Op OC, SPIRVDecorationGroup *TheGroup, const std::vector &TheTargets) : SPIRVEntryNoIdGeneric(TheGroup->getModule(), FixedWC + TheTargets.size(), OC), DecorationGroup(TheGroup), Targets(TheTargets) {} // Incomplete constructor SPIRVGroupDecorateGeneric(Op OC) : SPIRVEntryNoIdGeneric(OC), DecorationGroup(nullptr) {} void setWordCount(SPIRVWord WC) override { SPIRVEntryNoIdGeneric::setWordCount(WC); Targets.resize(WC - FixedWC); } virtual void decorateTargets() = 0; _SPIRV_DCL_ENCDEC protected: SPIRVDecorationGroup *DecorationGroup; std::vector Targets; }; class SPIRVGroupDecorate : public SPIRVGroupDecorateGeneric { public: static const Op OC = OpGroupDecorate; // Complete constructor SPIRVGroupDecorate(SPIRVDecorationGroup *TheGroup, const std::vector &TheTargets) : SPIRVGroupDecorateGeneric(OC, TheGroup, TheTargets) {} // Incomplete constructor SPIRVGroupDecorate() : SPIRVGroupDecorateGeneric(OC) {} void decorateTargets() override; }; class SPIRVGroupMemberDecorate : public SPIRVGroupDecorateGeneric { public: static const Op OC = OpGroupMemberDecorate; // Complete constructor SPIRVGroupMemberDecorate(SPIRVDecorationGroup *TheGroup, const std::vector &TheTargets) : SPIRVGroupDecorateGeneric(OC, TheGroup, TheTargets) {} // Incomplete constructor SPIRVGroupMemberDecorate() : SPIRVGroupDecorateGeneric(OC) {} void decorateTargets() override; }; template class SPIRVDecorateStrAttrBase : public SPIRVDecorate { public: // Complete constructor for decoration with string literal SPIRVDecorateStrAttrBase(SPIRVEntry *TheTarget, const std::string &Str) : SPIRVDecorate(D, TheTarget) { for (auto &I : getVec(Str)) Literals.push_back(I); WordCount += Literals.size(); } // Incomplete constructor SPIRVDecorateStrAttrBase() : SPIRVDecorate() {} static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << getString(Literals.cbegin(), Literals.cend()); } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { std::string Str; Decoder >> Str; std::copy_n(getVec(Str).begin(), Literals.size(), Literals.begin()); } else #endif Decoder >> Literals; } }; class SPIRVDecorateMemoryINTELAttr : public SPIRVDecorateStrAttrBase { public: // Complete constructor for MemoryINTEL decoration SPIRVDecorateMemoryINTELAttr(SPIRVEntry *TheTarget, const std::string &MemoryType) : SPIRVDecorateStrAttrBase(TheTarget, MemoryType) {} }; class SPIRVDecorateUserSemanticAttr : public SPIRVDecorateStrAttrBase { public: // Complete constructor for UserSemantic decoration SPIRVDecorateUserSemanticAttr(SPIRVEntry *TheTarget, const std::string &AnnotateString) : SPIRVDecorateStrAttrBase(TheTarget, AnnotateString) {} }; class SPIRVDecorateMergeINTELAttr : public SPIRVDecorate { public: // Complete constructor for MergeINTEL decoration SPIRVDecorateMergeINTELAttr(SPIRVEntry *TheTarget, const std::string &Name, const std::string &Direction) : SPIRVDecorate(DecorationMergeINTEL, TheTarget) { for (auto &I : getVec(Name)) Literals.push_back(I); for (auto &I : getVec(Direction)) Literals.push_back(I); WordCount += Literals.size(); } static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { std::string FirstString = getString(Literals.cbegin(), Literals.cend()); Encoder << FirstString; Encoder.OS << " "; Encoder << getString(Literals.cbegin() + getVec(FirstString).size(), Literals.cend()); } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { std::string Name; Decoder >> Name; std::string Direction; Decoder >> Direction; std::string Buf = Name + ':' + Direction; std::copy_n(getVec(Buf).begin(), Literals.size(), Literals.begin()); } else #endif Decoder >> Literals; } }; class SPIRVDecorateBankBitsINTELAttr : public SPIRVDecorate { public: // Complete constructor for BankBitsINTEL decoration SPIRVDecorateBankBitsINTELAttr(SPIRVEntry *TheTarget, const std::vector &TheBits) : SPIRVDecorate(DecorationBankBitsINTEL, TheTarget) { Literals = TheBits; WordCount += Literals.size(); } }; template class SPIRVMemberDecorateStrAttrBase : public SPIRVMemberDecorate { public: // Complete constructor for decoration with string literal SPIRVMemberDecorateStrAttrBase(SPIRVEntry *TheTarget, SPIRVWord MemberNumber, const std::string &Str) : SPIRVMemberDecorate(D, MemberNumber, TheTarget) { for (auto &I : getVec(Str)) Literals.push_back(I); WordCount += Literals.size(); } // Incomplete constructor SPIRVMemberDecorateStrAttrBase() : SPIRVMemberDecorate() {} }; class SPIRVMemberDecorateMemoryINTELAttr : public SPIRVMemberDecorateStrAttrBase { public: // Complete constructor for MemoryINTEL decoration SPIRVMemberDecorateMemoryINTELAttr(SPIRVEntry *TheTarget, SPIRVWord MemberNumber, const std::string &MemoryType) : SPIRVMemberDecorateStrAttrBase(TheTarget, MemberNumber, MemoryType) {} }; class SPIRVMemberDecorateUserSemanticAttr : public SPIRVMemberDecorateStrAttrBase { public: // Complete constructor for UserSemantic decoration SPIRVMemberDecorateUserSemanticAttr(SPIRVEntry *TheTarget, SPIRVWord MemberNumber, const std::string &AnnotateString) : SPIRVMemberDecorateStrAttrBase(TheTarget, MemberNumber, AnnotateString) {} }; class SPIRVMemberDecorateMergeINTELAttr : public SPIRVMemberDecorate { public: // Complete constructor for MergeINTEL decoration SPIRVMemberDecorateMergeINTELAttr(SPIRVEntry *TheTarget, SPIRVWord MemberNumber, const std::string &Name, const std::string &Direction) : SPIRVMemberDecorate(DecorationMergeINTEL, MemberNumber, TheTarget) { for (auto &I : getVec(Name)) Literals.push_back(I); for (auto &I : getVec(Direction)) Literals.push_back(I); WordCount += Literals.size(); } }; class SPIRVMemberDecorateBankBitsINTELAttr : public SPIRVMemberDecorate { public: // Complete constructor for BankBitsINTEL decoration SPIRVMemberDecorateBankBitsINTELAttr(SPIRVEntry *TheTarget, SPIRVWord MemberNumber, const std::vector &TheBits) : SPIRVMemberDecorate(DecorationBankBitsINTEL, MemberNumber, TheTarget) { Literals = TheBits; WordCount += Literals.size(); } }; class SPIRVDecorateFunctionRoundingModeINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateFunctionRoundingModeINTEL SPIRVDecorateFunctionRoundingModeINTEL(SPIRVEntry *TheTarget, SPIRVWord TargetWidth, spv::FPRoundingMode FloatControl) : SPIRVDecorate(spv::DecorationFunctionRoundingModeINTEL, TheTarget, TargetWidth, static_cast(FloatControl)){}; SPIRVWord getTargetWidth() const { return Literals.at(0); }; spv::FPRoundingMode getRoundingMode() const { return static_cast(Literals.at(1)); }; }; class SPIRVDecorateFunctionDenormModeINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateFunctionDenormModeINTEL SPIRVDecorateFunctionDenormModeINTEL(SPIRVEntry *TheTarget, SPIRVWord TargetWidth, spv::FPDenormMode FloatControl) : SPIRVDecorate(spv::DecorationFunctionDenormModeINTEL, TheTarget, TargetWidth, static_cast(FloatControl)){}; SPIRVWord getTargetWidth() const { return Literals.at(0); }; spv::FPDenormMode getDenormMode() const { return static_cast(Literals.at(1)); }; }; class SPIRVDecorateFunctionFloatingPointModeINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateFunctionOperationModeINTEL SPIRVDecorateFunctionFloatingPointModeINTEL(SPIRVEntry *TheTarget, SPIRVWord TargetWidth, spv::FPOperationMode FloatControl) : SPIRVDecorate(spv::DecorationFunctionFloatingPointModeINTEL, TheTarget, TargetWidth, static_cast(FloatControl)){}; SPIRVWord getTargetWidth() const { return Literals.at(0); }; spv::FPOperationMode getOperationMode() const { return static_cast(Literals.at(1)); }; }; class SPIRVDecorateStallEnableINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateStallEnableINTEL SPIRVDecorateStallEnableINTEL(SPIRVEntry *TheTarget) : SPIRVDecorate(spv::DecorationStallEnableINTEL, TheTarget){}; }; class SPIRVDecorateFuseLoopsInFunctionINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateFuseLoopsInFunctionINTEL SPIRVDecorateFuseLoopsInFunctionINTEL(SPIRVEntry *TheTarget, SPIRVWord Depth, SPIRVWord Independent) : SPIRVDecorate(spv::DecorationFuseLoopsInFunctionINTEL, TheTarget, Depth, Independent){}; }; class SPIRVDecorateMathOpDSPModeINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateMathOpDSPModeINTEL SPIRVDecorateMathOpDSPModeINTEL(SPIRVEntry *TheTarget, SPIRVWord Mode, SPIRVWord Propagate) : SPIRVDecorate(spv::internal::DecorationMathOpDSPModeINTEL, TheTarget, Mode, Propagate){}; }; class SPIRVDecorateAliasScopeINTEL : public SPIRVDecorateId { public: // Complete constructor for SPIRVDecorateAliasScopeINTEL SPIRVDecorateAliasScopeINTEL(SPIRVEntry *TheTarget, SPIRVId AliasList) : SPIRVDecorateId(spv::DecorationAliasScopeINTEL, TheTarget, AliasList){}; }; class SPIRVDecorateNoAliasINTEL : public SPIRVDecorateId { public: // Complete constructor for SPIRVDecorateNoAliasINTEL SPIRVDecorateNoAliasINTEL(SPIRVEntry *TheTarget, SPIRVId AliasList) : SPIRVDecorateId(spv::DecorationNoAliasINTEL, TheTarget, AliasList){}; }; class SPIRVDecorateInitiationIntervalINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateInitiationIntervalINTEL SPIRVDecorateInitiationIntervalINTEL(SPIRVEntry *TheTarget, SPIRVWord Cycles) : SPIRVDecorate(spv::internal::DecorationInitiationIntervalINTEL, TheTarget, Cycles){}; }; class SPIRVDecorateMaxConcurrencyINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateMaxConcurrencyINTEL SPIRVDecorateMaxConcurrencyINTEL(SPIRVEntry *TheTarget, SPIRVWord Invocations) : SPIRVDecorate(spv::internal::DecorationMaxConcurrencyINTEL, TheTarget, Invocations){}; }; class SPIRVDecoratePipelineEnableINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecoratePipelineEnableINTEL SPIRVDecoratePipelineEnableINTEL(SPIRVEntry *TheTarget, SPIRVWord Enable) : SPIRVDecorate(spv::internal::DecorationPipelineEnableINTEL, TheTarget, Enable){}; }; class SPIRVDecorateHostAccessINTELBase : public SPIRVDecorate { public: // Complete constructor for SPIRVHostAccessINTEL SPIRVDecorateHostAccessINTELBase(Decoration D, SPIRVEntry *TheTarget, HostAccessQualifier AccessMode, const std::string &VarName) : SPIRVDecorate(D, TheTarget) { Literals.push_back(AccessMode); for (auto &I : getVec(VarName)) Literals.push_back(I); WordCount += Literals.size(); }; SPIRVWord getAccessMode() const { return Literals.front(); } std::string getVarName() const { return getString(Literals.cbegin() + 1, Literals.cend()); } }; class SPIRVDecorateHostAccessINTEL : public SPIRVDecorateHostAccessINTELBase { public: SPIRVDecorateHostAccessINTEL(SPIRVEntry *TheTarget, HostAccessQualifier AccessMode, const std::string &VarName) : SPIRVDecorateHostAccessINTELBase(DecorationHostAccessINTEL, TheTarget, AccessMode, VarName) {} static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << (HostAccessQualifier)Literals.front(); std::string Name = getString(Literals.cbegin() + 1, Literals.cend()); Encoder << Name; } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { HostAccessQualifier Mode; Decoder >> Mode; std::string Name; Decoder >> Name; Literals.front() = Mode; std::copy_n(getVec(Name).begin(), Literals.size() - 1, Literals.begin() + 1); } else #endif Decoder >> Literals; } }; class SPIRVDecorateHostAccessINTELLegacy : public SPIRVDecorateHostAccessINTELBase { public: SPIRVDecorateHostAccessINTELLegacy(SPIRVEntry *TheTarget, HostAccessQualifier AccessMode, const std::string &VarName) : SPIRVDecorateHostAccessINTELBase(internal::DecorationHostAccessINTEL, TheTarget, AccessMode, VarName) {} static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << Literals.front(); std::string Name = getString(Literals.cbegin() + 1, Literals.cend()); Encoder << Name; } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { SPIRVWord Mode; Decoder >> Mode; std::string Name; Decoder >> Name; Literals.front() = Mode; std::copy_n(getVec(Name).begin(), Literals.size() - 1, Literals.begin() + 1); } else #endif Decoder >> Literals; } }; class SPIRVDecorateInitModeINTELBase : public SPIRVDecorate { public: // Complete constructor for SPIRVInitModeINTEL SPIRVDecorateInitModeINTELBase(Decoration D, SPIRVEntry *TheTarget, InitializationModeQualifier Trigger) : SPIRVDecorate(D, TheTarget) { Literals.push_back(Trigger); WordCount += Literals.size(); } }; class SPIRVDecorateInitModeINTEL : public SPIRVDecorateInitModeINTELBase { public: SPIRVDecorateInitModeINTEL(SPIRVEntry *TheTarget, InitializationModeQualifier Trigger) : SPIRVDecorateInitModeINTELBase(DecorationInitModeINTEL, TheTarget, Trigger) {} static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << (InitializationModeQualifier)Literals.back(); } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { InitializationModeQualifier Q; Decoder >> Q; Literals.back() = Q; } else #endif Decoder >> Literals; } }; class SPIRVDecorateInitModeINTELLegacy : public SPIRVDecorateInitModeINTELBase { public: SPIRVDecorateInitModeINTELLegacy(SPIRVEntry *TheTarget, InitializationModeQualifier Trigger) : SPIRVDecorateInitModeINTELBase(internal::DecorationInitModeINTEL, TheTarget, Trigger) {} static void encodeLiterals(SPIRVEncoder &Encoder, const std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { Encoder << Literals.back(); } else #endif Encoder << Literals; } static void decodeLiterals(SPIRVDecoder &Decoder, std::vector &Literals) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { SPIRVWord Q; Decoder >> Q; Literals.back() = Q; } else #endif Decoder >> Literals; } }; class SPIRVDecorateImplementInCSRINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVImplementInCSRINTEL SPIRVDecorateImplementInCSRINTEL(SPIRVEntry *TheTarget, SPIRVWord Value) : SPIRVDecorate(spv::internal::DecorationImplementInCSRINTEL, TheTarget, Value) {} }; class SPIRVDecorateImplementInRegisterMapINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVImplementInCSRINTEL SPIRVDecorateImplementInRegisterMapINTEL(SPIRVEntry *TheTarget, SPIRVWord Value) : SPIRVDecorate(DecorationImplementInRegisterMapINTEL, TheTarget, Value) { } }; class SPIRVDecorateCacheControlLoadINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateCacheControlLoadINTEL SPIRVDecorateCacheControlLoadINTEL(SPIRVEntry *TheTarget, SPIRVWord CacheLevel, LoadCacheControl CacheControl) : SPIRVDecorate(DecorationCacheControlLoadINTEL, TheTarget, CacheLevel, static_cast(CacheControl)) {} SPIRVWord getCacheLevel() const { return Literals.at(0); } LoadCacheControl getCacheControl() const { return static_cast(Literals.at(1)); } }; class SPIRVDecorateCacheControlStoreINTEL : public SPIRVDecorate { public: // Complete constructor for SPIRVDecorateCacheControlStoreINTEL SPIRVDecorateCacheControlStoreINTEL(SPIRVEntry *TheTarget, SPIRVWord CacheLevel, StoreCacheControl CacheControl) : SPIRVDecorate(DecorationCacheControlStoreINTEL, TheTarget, CacheLevel, static_cast(CacheControl)) {} SPIRVWord getCacheLevel() const { return Literals.at(0); } StoreCacheControl getCacheControl() const { return static_cast(Literals.at(1)); } }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVDECORATE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVEntry.cpp000066400000000000000000000637241506646577400232230ustar00rootroot00000000000000//===- SPIRVEntry.cpp - Base Class for SPIR-V Entities ----------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements base class for SPIR-V entities. /// //===----------------------------------------------------------------------===// #include "SPIRVEntry.h" #include "SPIRVAsm.h" #include "SPIRVBasicBlock.h" #include "SPIRVDebug.h" #include "SPIRVDecorate.h" #include "SPIRVFnVar.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVMemAliasingINTEL.h" #include "SPIRVStream.h" #include "SPIRVType.h" #include #include #include #include #include #include using namespace SPIRV; namespace SPIRV { template SPIRVEntry *create() { return new T(); } SPIRVEntry *SPIRVEntry::create(Op OpCode) { typedef SPIRVEntry *(*SPIRVFactoryTy)(); struct TableEntry { Op Opn; SPIRVFactoryTy Factory; operator std::pair() { return std::make_pair(Opn, Factory); } }; static TableEntry Table[] = { #define _SPIRV_OP(x, ...) {Op##x, &SPIRV::create}, #define _SPIRV_OP_INTERNAL(x, ...) {internal::Op##x, &SPIRV::create}, #include "SPIRVOpCodeEnum.h" #include "SPIRVOpCodeEnumInternal.h" #undef _SPIRV_OP_INTERNAL #undef _SPIRV_OP }; typedef std::map OpToFactoryMapTy; static const OpToFactoryMapTy OpToFactoryMap(std::begin(Table), std::end(Table)); OpToFactoryMapTy::const_iterator Loc = OpToFactoryMap.find(OpCode); if (Loc != OpToFactoryMap.end()) return Loc->second(); SPIRVDBG(spvdbgs() << "No factory for OpCode " << (unsigned)OpCode << '\n';) assert(0 && "Not implemented"); return 0; } std::unique_ptr SPIRVEntry::createUnique(Op OC) { return std::unique_ptr(create(OC)); } std::unique_ptr SPIRVEntry::createUnique(SPIRVExtInstSetKind Set, unsigned ExtOp) { return std::unique_ptr(new SPIRVExtInst(Set, ExtOp)); } SPIRVErrorLog &SPIRVEntry::getErrorLog() const { return Module->getErrorLog(); } bool SPIRVEntry::exist(SPIRVId TheId) const { return Module->exist(TheId); } SPIRVEntry *SPIRVEntry::getOrCreate(SPIRVId TheId) const { SPIRVEntry *Entry = nullptr; bool Found = Module->exist(TheId, &Entry); if (!Found) return Module->addForward(TheId, nullptr); return Entry; } SPIRVValue *SPIRVEntry::getValue(SPIRVId TheId) const { return get(TheId); } SPIRVType *SPIRVEntry::getValueType(SPIRVId TheId) const { return get(TheId)->getType(); } SPIRVEncoder SPIRVEntry::getEncoder(spv_ostream &O) const { return SPIRVEncoder(O); } SPIRVDecoder SPIRVEntry::getDecoder(std::istream &I) { return SPIRVDecoder(I, *Module); } void SPIRVEntry::setWordCount(SPIRVWord TheWordCount) { WordCount = TheWordCount; } void SPIRVEntry::setName(const std::string &TheName) { Name = TheName; SPIRVDBG(spvdbgs() << "Set name for obj " << Id << " " << Name << '\n'); } void SPIRVEntry::setModule(SPIRVModule *TheModule) { assert(TheModule && "Invalid module"); if (TheModule == Module) return; assert(Module == NULL && "Cannot change owner of entry"); Module = TheModule; } void SPIRVEntry::encode(spv_ostream &O) const { assert(0 && "Not implemented"); } void SPIRVEntry::encodeName(spv_ostream &O) const { if (!Name.empty()) O << SPIRVName(this, Name); } bool SPIRVEntry::isEndOfBlock() const { switch (OpCode) { case OpBranch: case OpBranchConditional: case OpSwitch: case OpKill: case OpReturn: case OpReturnValue: case OpUnreachable: return true; default: return false; } } void SPIRVEntry::encodeLine(spv_ostream &O) const { if (!Module) return; const std::shared_ptr &CurrLine = Module->getCurrentLine(); if (Line && (!CurrLine || *Line != *CurrLine)) { O << *Line; Module->setCurrentLine(Line); } if (isEndOfBlock() || OpCode == OpNoLine) Module->setCurrentLine(nullptr); } namespace { bool isDebugLineEqual(const SPIRVExtInst &DL1, const SPIRVExtInst &DL2) { std::vector DL1Args = DL1.getArguments(); std::vector DL2Args = DL2.getArguments(); using namespace SPIRVDebug::Operand::DebugLine; assert(DL1Args.size() == OperandCount && DL2Args.size() == OperandCount && "Invalid number of operands"); return DL1Args[SourceIdx] == DL2Args[SourceIdx] && DL1Args[StartIdx] == DL2Args[StartIdx] && DL1Args[EndIdx] == DL2Args[EndIdx] && DL1Args[ColumnStartIdx] == DL2Args[ColumnStartIdx] && DL1Args[ColumnEndIdx] == DL2Args[ColumnEndIdx]; } } // namespace void SPIRVEntry::encodeDebugLine(spv_ostream &O) const { if (!Module) return; const std::shared_ptr &CurrDebugLine = Module->getCurrentDebugLine(); if (DebugLine && (!CurrDebugLine || !isDebugLineEqual(*DebugLine, *CurrDebugLine))) { O << *DebugLine; Module->setCurrentDebugLine(DebugLine); } if (isEndOfBlock() || isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::DebugNoLine) || isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugNoLine)) Module->setCurrentDebugLine(nullptr); } void SPIRVEntry::encodeAll(spv_ostream &O) const { encodeLine(O); encodeDebugLine(O); encodeWordCountOpCode(O); encode(O); encodeChildren(O); } void SPIRVEntry::encodeChildren(spv_ostream &O) const {} void SPIRVEntry::encodeWordCountOpCode(spv_ostream &O) const { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { getEncoder(O) << WordCount << OpCode; return; } #endif assert(WordCount < 65536 && "WordCount must fit into 16-bit value"); SPIRVWord WordCountOpCode = (WordCount << WordCountShift) | OpCode; getEncoder(O) << WordCountOpCode; } // Read words from SPIRV binary and create members for SPIRVEntry. // The word count and op code has already been read before calling this // function for creating the SPIRVEntry. Therefore the input stream only // contains the remaining part of the words for the SPIRVEntry. void SPIRVEntry::decode(std::istream &I) { assert(0 && "Not implemented"); } std::vector SPIRVEntry::getValues(const std::vector &IdVec) const { std::vector ValueVec; for (auto I : IdVec) ValueVec.push_back(getValue(I)); return ValueVec; } std::vector SPIRVEntry::getValueTypes(const std::vector &IdVec) const { std::vector TypeVec; for (auto I : IdVec) TypeVec.push_back(getValue(I)->getType()); return TypeVec; } std::vector SPIRVEntry::getIds(const std::vector ValueVec) const { std::vector IdVec; for (auto I : ValueVec) IdVec.push_back(I->getId()); return IdVec; } SPIRVEntry *SPIRVEntry::getEntry(SPIRVId TheId) const { return Module->getEntry(TheId); } void SPIRVEntry::validateFunctionControlMask(SPIRVWord TheFCtlMask) const { SPIRVCK(isValidFunctionControlMask(TheFCtlMask), InvalidFunctionControlMask, ""); } void SPIRVEntry::validateValues(const std::vector &Ids) const { for (auto I : Ids) getValue(I)->validate(); } void SPIRVEntry::validateBuiltin(SPIRVWord TheSet, SPIRVWord Index) const { assert(TheSet != SPIRVWORD_MAX && Index != SPIRVWORD_MAX && "Invalid builtin"); } void SPIRVEntry::addDecorate(SPIRVDecorate *Dec) { auto Kind = Dec->getDecorateKind(); Decorates.insert(std::make_pair(Kind, Dec)); Module->addDecorate(Dec); if (Kind == spv::DecorationLinkageAttributes) { auto *LinkageAttr = static_cast(Dec); setName(LinkageAttr->getLinkageName()); } SPIRVDBG(spvdbgs() << "[addDecorate] " << *Dec << '\n';) } void SPIRVEntry::addDecorate(SPIRVDecorateId *Dec) { DecorateIds.insert(std::make_pair(Dec->getDecorateKind(), Dec)); Module->addDecorate(Dec); SPIRVDBG(spvdbgs() << "[addDecorateId] " << *Dec << '\n';) } void SPIRVEntry::addDecorate(Decoration Kind) { addDecorate(new SPIRVDecorate(Kind, this)); } void SPIRVEntry::addDecorate(Decoration Kind, SPIRVWord Literal) { switch (static_cast(Kind)) { case DecorationAliasScopeINTEL: case DecorationNoAliasINTEL: addDecorate(new SPIRVDecorateId(Kind, this, Literal)); return; default: addDecorate(new SPIRVDecorate(Kind, this, Literal)); } } void SPIRVEntry::eraseDecorate(Decoration Dec) { Decorates.erase(Dec); } void SPIRVEntry::takeDecorates(SPIRVEntry *E) { Decorates = std::move(E->Decorates); SPIRVDBG(spvdbgs() << "[takeDecorates] " << Id << '\n';) } void SPIRVEntry::takeDecorateIds(SPIRVEntry *E) { DecorateIds = std::move(E->DecorateIds); SPIRVDBG(spvdbgs() << "[takeDecorateIds] " << Id << '\n';) } void SPIRVEntry::setLine(const std::shared_ptr &L) { Line = L; SPIRVDBG(if (L) spvdbgs() << "[setLine] " << *L << '\n';) } void SPIRVEntry::setDebugLine(const std::shared_ptr &DL) { DebugLine = DL; SPIRVDBG(if (DL) spvdbgs() << "[setDebugLine] " << *DL << '\n';) } void SPIRVEntry::addMemberDecorate(SPIRVMemberDecorate *Dec) { assert(canHaveMemberDecorates() && MemberDecorates.find(Dec->getPair()) == MemberDecorates.end()); MemberDecorates[Dec->getPair()] = Dec; Module->addDecorate(Dec); SPIRVDBG(spvdbgs() << "[addMemberDecorate] " << *Dec << '\n';) } void SPIRVEntry::addMemberDecorate(SPIRVWord MemberNumber, Decoration Kind) { addMemberDecorate(new SPIRVMemberDecorate(Kind, MemberNumber, this)); } void SPIRVEntry::addMemberDecorate(SPIRVWord MemberNumber, Decoration Kind, SPIRVWord Literal) { addMemberDecorate(new SPIRVMemberDecorate(Kind, MemberNumber, this, Literal)); } void SPIRVEntry::eraseMemberDecorate(SPIRVWord MemberNumber, Decoration Dec) { MemberDecorates.erase(std::make_pair(MemberNumber, Dec)); } void SPIRVEntry::takeMemberDecorates(SPIRVEntry *E) { MemberDecorates = std::move(E->MemberDecorates); SPIRVDBG(spvdbgs() << "[takeMemberDecorates] " << Id << '\n';) } void SPIRVEntry::takeAnnotations(SPIRVForward *E) { Module->setName(this, E->getName()); takeDecorates(E); takeDecorateIds(E); takeMemberDecorates(E); if (OpCode == OpFunction) static_cast(this)->takeExecutionModes(E); } void SPIRVEntry::replaceTargetIdInDecorates(SPIRVId Id) { for (auto It = Decorates.begin(), E = Decorates.end(); It != E; ++It) const_cast(It->second)->setTargetId(Id); for (auto It = DecorateIds.begin(), E = DecorateIds.end(); It != E; ++It) const_cast(It->second)->setTargetId(Id); for (auto It = MemberDecorates.begin(), E = MemberDecorates.end(); It != E; ++It) const_cast(It->second)->setTargetId(Id); } // Check if an entry has Kind of decoration and get the literal of the // first decoration of such kind at Index. bool SPIRVEntry::hasDecorate(Decoration Kind, size_t Index, SPIRVWord *Result) const { auto Loc = Decorates.find(Kind); if (Loc == Decorates.end()) return false; if (Result) *Result = Loc->second->getLiteral(Index); return true; } bool SPIRVEntry::hasDecorateId(Decoration Kind, size_t Index, SPIRVId *Result) const { auto Loc = DecorateIds.find(Kind); if (Loc == DecorateIds.end()) return false; if (Result) *Result = Loc->second->getLiteral(Index); return true; } // Check if an entry member has Kind of decoration and get the literal of the // first decoration of such kind at Index. bool SPIRVEntry::hasMemberDecorate(Decoration Kind, size_t Index, SPIRVWord MemberNumber, SPIRVWord *Result) const { auto Loc = MemberDecorates.find({MemberNumber, Kind}); if (Loc == MemberDecorates.end()) return false; if (Result) *Result = Loc->second->getLiteral(Index); return true; } std::vector SPIRVEntry::getDecorationStringLiteral(Decoration Kind) const { auto Loc = Decorates.find(Kind); if (Loc == Decorates.end()) return {}; return getVecString(Loc->second->getVecLiteral()); } std::vector SPIRVEntry::getMemberDecorationStringLiteral(Decoration Kind, SPIRVWord MemberNumber) const { auto Loc = MemberDecorates.find({MemberNumber, Kind}); if (Loc == MemberDecorates.end()) return {}; return getVecString(Loc->second->getVecLiteral()); } std::vector SPIRVEntry::getDecorationLiterals(Decoration Kind) const { auto Loc = Decorates.find(Kind); if (Loc == Decorates.end()) return {}; return (Loc->second->getVecLiteral()); } std::vector SPIRVEntry::getDecorationIdLiterals(Decoration Kind) const { auto Loc = DecorateIds.find(Kind); if (Loc == DecorateIds.end()) return {}; return (Loc->second->getVecLiteral()); } std::vector SPIRVEntry::getMemberDecorationLiterals(Decoration Kind, SPIRVWord MemberNumber) const { auto Loc = MemberDecorates.find({MemberNumber, Kind}); if (Loc == MemberDecorates.end()) return {}; return (Loc->second->getVecLiteral()); } // Get literals of all decorations of Kind at Index. std::set SPIRVEntry::getDecorate(Decoration Kind, size_t Index) const { auto Range = Decorates.equal_range(Kind); std::set Value; for (auto I = Range.first, E = Range.second; I != E; ++I) { assert(Index < I->second->getLiteralCount() && "Invalid index"); Value.insert(I->second->getLiteral(Index)); } return Value; } std::vector SPIRVEntry::getDecorations(Decoration Kind) const { auto Range = Decorates.equal_range(Kind); std::vector Decors; Decors.reserve(Decorates.count(Kind)); for (auto I = Range.first, E = Range.second; I != E; ++I) { Decors.push_back(I->second); } return Decors; } std::vector SPIRVEntry::getDecorations() const { std::vector Decors; Decors.reserve(Decorates.size()); for (auto &DecoPair : Decorates) Decors.push_back(DecoPair.second); return Decors; } std::set SPIRVEntry::getDecorateId(Decoration Kind, size_t Index) const { auto Range = DecorateIds.equal_range(Kind); std::set Value; for (auto I = Range.first, E = Range.second; I != E; ++I) { assert(Index < I->second->getLiteralCount() && "Invalid index"); Value.insert(I->second->getLiteral(Index)); } return Value; } std::vector SPIRVEntry::getDecorationIds(Decoration Kind) const { auto Range = DecorateIds.equal_range(Kind); std::vector Decors; Decors.reserve(DecorateIds.count(Kind)); for (auto I = Range.first, E = Range.second; I != E; ++I) { Decors.push_back(I->second); } return Decors; } bool SPIRVEntry::hasLinkageType() const { return OpCode == OpFunction || OpCode == OpVariable; } bool SPIRVEntry::isExtInst(const SPIRVExtInstSetKind InstSet) const { if (isExtInst()) { const SPIRVExtInst *EI = static_cast(this); return EI->getExtSetKind() == InstSet; } return false; } bool SPIRVEntry::isExtInst(const SPIRVExtInstSetKind InstSet, const SPIRVWord ExtOp) const { if (isExtInst()) { const SPIRVExtInst *EI = static_cast(this); if (EI->getExtSetKind() == InstSet) { return EI->getExtOp() == ExtOp; } } return false; } void SPIRVEntry::encodeDecorate(spv_ostream &O) const { for (auto &I : Decorates) O << *I.second; for (auto &I : DecorateIds) O << *I.second; } SPIRVLinkageTypeKind SPIRVEntry::getLinkageType() const { assert(hasLinkageType()); DecorateMapType::const_iterator Loc = Decorates.find(DecorationLinkageAttributes); if (Loc == Decorates.end()) return internal::LinkageTypeInternal; return static_cast(Loc->second) ->getLinkageType(); } void SPIRVEntry::setLinkageType(SPIRVLinkageTypeKind LT) { assert(isValid(LT)); assert(hasLinkageType()); addDecorate(new SPIRVDecorateLinkageAttr(this, Name, LT)); } void SPIRVEntry::updateModuleVersion() const { if (!Module) return; Module->setMinSPIRVVersion( static_cast(getRequiredSPIRVVersion())); } spv_ostream &operator<<(spv_ostream &O, const SPIRVEntry &E) { E.validate(); E.encodeAll(O); O << SPIRVNL(); return O; } std::istream &operator>>(std::istream &I, SPIRVEntry &E) { E.decode(I); return I; } SPIRVEntryPoint::SPIRVEntryPoint(SPIRVModule *TheModule, SPIRVExecutionModelKind TheExecModel, SPIRVId TheId, const std::string &TheName, std::vector Variables) : SPIRVAnnotation(OpEntryPoint, TheModule->get(TheId), getSizeInWords(TheName) + Variables.size() + 3), ExecModel(TheExecModel), Name(TheName), Variables(Variables) {} void SPIRVEntryPoint::encode(spv_ostream &O) const { getEncoder(O) << ExecModel << Target << Name << Variables; } void SPIRVEntryPoint::decode(std::istream &I) { getDecoder(I) >> ExecModel >> Target >> Name; Variables.resize(WordCount - FixedWC - getSizeInWords(Name) + 1); getDecoder(I) >> Variables; Module->setName(getOrCreateTarget(), Name); Module->addEntryPoint(ExecModel, Target, Name, Variables); } void SPIRVExecutionMode::encode(spv_ostream &O) const { getEncoder(O) << Target << ExecMode << WordLiterals; } void SPIRVExecutionMode::decode(std::istream &I) { getDecoder(I) >> Target >> ExecMode; switch (static_cast(ExecMode)) { case ExecutionModeLocalSize: case ExecutionModeLocalSizeId: case ExecutionModeLocalSizeHint: case ExecutionModeLocalSizeHintId: case ExecutionModeMaxWorkgroupSizeINTEL: WordLiterals.resize(3); break; case ExecutionModeInvocations: case ExecutionModeOutputVertices: case ExecutionModeVecTypeHint: case ExecutionModeDenormPreserve: case ExecutionModeDenormFlushToZero: case ExecutionModeSignedZeroInfNanPreserve: case ExecutionModeRoundingModeRTE: case ExecutionModeRoundingModeRTZ: case ExecutionModeRoundingModeRTPINTEL: case ExecutionModeRoundingModeRTNINTEL: case ExecutionModeFloatingPointModeALTINTEL: case ExecutionModeFloatingPointModeIEEEINTEL: case ExecutionModeSharedLocalMemorySizeINTEL: case ExecutionModeNamedBarrierCountINTEL: case ExecutionModeSubgroupSize: case ExecutionModeSubgroupsPerWorkgroup: case ExecutionModeSubgroupsPerWorkgroupId: case ExecutionModeMaxWorkDimINTEL: case ExecutionModeNumSIMDWorkitemsINTEL: case ExecutionModeSchedulerTargetFmaxMhzINTEL: case ExecutionModeStreamingInterfaceINTEL: case ExecutionModeMaximumRegistersINTEL: case ExecutionModeMaximumRegistersIdINTEL: case ExecutionModeNamedMaximumRegistersINTEL: WordLiterals.resize(1); break; default: // Do nothing. Keep this to avoid VS2013 warning. break; } getDecoder(I) >> WordLiterals; getOrCreateTarget()->addExecutionMode(Module->add(this)); } SPIRVForward *SPIRVAnnotationGeneric::getOrCreateTarget() const { SPIRVEntry *Entry = nullptr; bool Found = Module->exist(Target, &Entry); assert((!Found || Entry->getOpCode() == internal::OpForward) && "Annotations only allowed on forward"); if (!Found) Entry = Module->addForward(Target, nullptr); return static_cast(Entry); } SPIRVName::SPIRVName(const SPIRVEntry *TheTarget, const std::string &TheStr) : SPIRVAnnotation(OpName, TheTarget, getSizeInWords(TheStr) + 2), Str(TheStr) {} void SPIRVName::encode(spv_ostream &O) const { getEncoder(O) << Target << Str; } void SPIRVName::decode(std::istream &I) { getDecoder(I) >> Target >> Str; Module->setName(getOrCreateTarget(), Str); } void SPIRVName::validate() const { assert(WordCount == getSizeInWords(Str) + 2 && "Incorrect word count"); } _SPIRV_IMP_ENCDEC2(SPIRVString, Id, Str) _SPIRV_IMP_ENCDEC3(SPIRVMemberName, Target, MemberNumber, Str) void SPIRVLine::encode(spv_ostream &O) const { getEncoder(O) << FileName << Line << Column; } void SPIRVLine::decode(std::istream &I) { getDecoder(I) >> FileName >> Line >> Column; } void SPIRVLine::validate() const { assert(OpCode == OpLine); assert(WordCount == 4); assert(get(FileName)->getOpCode() == OpString); assert(Line != SPIRVWORD_MAX); assert(Column != SPIRVWORD_MAX); assert(!hasId()); } void SPIRVMemberName::validate() const { assert(OpCode == OpMemberName); assert(WordCount == getSizeInWords(Str) + FixedWC); assert(get(Target)->getOpCode() == OpTypeStruct); assert(MemberNumber < get(Target)->getStructMemberCount()); } SPIRVExtInstImport::SPIRVExtInstImport(SPIRVModule *TheModule, SPIRVId TheId, const std::string &TheStr) : SPIRVEntry(TheModule, 2 + getSizeInWords(TheStr), OC, TheId), Str(TheStr) { validate(); } void SPIRVExtInstImport::encode(spv_ostream &O) const { getEncoder(O) << Id << Str; } void SPIRVExtInstImport::decode(std::istream &I) { getDecoder(I) >> Id >> Str; Module->importBuiltinSetWithId(Str, Id); } void SPIRVExtInstImport::validate() const { SPIRVEntry::validate(); assert(!Str.empty() && "Invalid builtin set"); } void SPIRVMemoryModel::encode(spv_ostream &O) const { getEncoder(O) << Module->getAddressingModel() << Module->getMemoryModel(); } void SPIRVMemoryModel::decode(std::istream &I) { SPIRVAddressingModelKind AddrModel; SPIRVMemoryModelKind MemModel; getDecoder(I) >> AddrModel >> MemModel; Module->setAddressingModel(AddrModel); Module->setMemoryModel(MemModel); } void SPIRVMemoryModel::validate() const { auto AM = Module->getAddressingModel(); auto MM = Module->getMemoryModel(); SPIRVCK(isValid(AM), InvalidAddressingModel, "Actual is " + std::to_string(AM)); SPIRVCK(isValid(MM), InvalidMemoryModel, "Actual is " + std::to_string(MM)); } void SPIRVSource::encode(spv_ostream &O) const { SPIRVWord Ver = SPIRVWORD_MAX; auto Language = Module->getSourceLanguage(&Ver); getEncoder(O) << Language << Ver; } void SPIRVSource::decode(std::istream &I) { SourceLanguage Lang = SourceLanguageUnknown; SPIRVWord Ver = SPIRVWORD_MAX; getDecoder(I) >> Lang >> Ver; Module->setSourceLanguage(Lang, Ver); } SPIRVSourceExtension::SPIRVSourceExtension(SPIRVModule *M, const std::string &SS) : SPIRVEntryNoId(M, 1 + getSizeInWords(SS)), S(SS) {} void SPIRVSourceExtension::encode(spv_ostream &O) const { getEncoder(O) << S; } void SPIRVSourceExtension::decode(std::istream &I) { getDecoder(I) >> S; Module->getSourceExtension().insert(S); } SPIRVExtension::SPIRVExtension(SPIRVModule *M, const std::string &SS) : SPIRVEntryNoId(M, 1 + getSizeInWords(SS)), S(SS) {} void SPIRVExtension::encode(spv_ostream &O) const { getEncoder(O) << S; } void SPIRVExtension::decode(std::istream &I) { getDecoder(I) >> S; Module->getExtension().insert(S); } SPIRVCapability::SPIRVCapability(SPIRVModule *M, SPIRVCapabilityKind K) : SPIRVEntryNoId(M, 2), Kind(K) { updateModuleVersion(); } void SPIRVCapability::encode(spv_ostream &O) const { getEncoder(O) << Kind; } void SPIRVCapability::decode(std::istream &I) { getDecoder(I) >> Kind; Module->addCapability(Kind); } template void SPIRVContinuedInstINTELBase::validate() const { SPIRVEntry::validate(); } template void SPIRVContinuedInstINTELBase::encode(spv_ostream &O) const { SPIRVEntry::getEncoder(O) << (Elements); } template void SPIRVContinuedInstINTELBase::decode(std::istream &I) { SPIRVEntry::getDecoder(I) >> (Elements); } SPIRVType *SPIRVTypeStructContinuedINTEL::getMemberType(size_t I) const { return static_cast(SPIRVEntry::getEntry(Elements[I])); } void SPIRVModuleProcessed::validate() const { assert(WordCount == FixedWC + getSizeInWords(ProcessStr) && "Incorrect word count in OpModuleProcessed"); } void SPIRVModuleProcessed::encode(spv_ostream &O) const { getEncoder(O) << ProcessStr; } void SPIRVModuleProcessed::decode(std::istream &I) { getDecoder(I) >> ProcessStr; Module->addModuleProcessed(ProcessStr); } std::string SPIRVModuleProcessed::getProcessStr() { return ProcessStr; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVEntry.h000066400000000000000000001241331506646577400226600ustar00rootroot00000000000000//===- SPIRVEntry.h - Base Class for SPIR-V Entities ------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the base class for SPIRV entities. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVENTRY_H #define SPIRV_LIBSPIRV_SPIRVENTRY_H #include "LLVMSPIRVOpts.h" #include "SPIRVEnum.h" #include "SPIRVError.h" #include "SPIRVIsValidEnum.h" #include #include #include #include #include #include #include #include namespace SPIRV { class SPIRVModule; class SPIRVEncoder; class SPIRVDecoder; class SPIRVType; class SPIRVValue; class SPIRVDecorate; class SPIRVDecorateId; class SPIRVForward; class SPIRVMemberDecorate; class SPIRVLine; class SPIRVString; class SPIRVExtInst; // Add declaration of encode/decode functions to a class. // Used inside class definition. #define _SPIRV_DCL_ENCDEC \ void encode(spv_ostream &O) const override; \ void decode(std::istream &I) override; #define _REQ_SPIRV_VER(Version) \ SPIRVWord getRequiredSPIRVVersion() const override { return Version; } // Add implementation of encode/decode functions to a class. // Used out side of class definition. #define _SPIRV_IMP_ENCDEC0(Ty) \ void Ty::encode(spv_ostream &O) const {} \ void Ty::decode(std::istream &I) {} #define _SPIRV_IMP_ENCDEC1(Ty, x) \ void Ty::encode(spv_ostream &O) const { getEncoder(O) << (x); } \ void Ty::decode(std::istream &I) { getDecoder(I) >> (x); } #define _SPIRV_IMP_ENCDEC2(Ty, x, y) \ void Ty::encode(spv_ostream &O) const { getEncoder(O) << (x) << (y); } \ void Ty::decode(std::istream &I) { getDecoder(I) >> (x) >> (y); } #define _SPIRV_IMP_ENCDEC3(Ty, x, y, z) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z); \ } \ void Ty::decode(std::istream &I) { getDecoder(I) >> (x) >> (y) >> (z); } #define _SPIRV_IMP_ENCDEC4(Ty, x, y, z, u) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u); \ } #define _SPIRV_IMP_ENCDEC5(Ty, x, y, z, u, v) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v); \ } #define _SPIRV_IMP_ENCDEC6(Ty, x, y, z, u, v, w) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w); \ } #define _SPIRV_IMP_ENCDEC7(Ty, x, y, z, u, v, w, r) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r); \ } #define _SPIRV_IMP_ENCDEC8(Ty, x, y, z, u, v, w, r, s) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r) << (s); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r) >> (s); \ } #define _SPIRV_IMP_ENCDEC9(Ty, x, y, z, u, v, w, r, s, t) \ void Ty::encode(spv_ostream &O) const { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r) << (s) \ << (t); \ } \ void Ty::decode(std::istream &I) { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r) >> (s) >> \ (t); \ } // Add definition of encode/decode functions to a class. // Used inside class definition. #define _SPIRV_DEF_ENCDEC0 \ void encode(spv_ostream &O) const override {} \ void decode(std::istream &I) override {} #define _SPIRV_DEF_ENCDEC1(x) \ void encode(spv_ostream &O) const override { getEncoder(O) << (x); } \ void decode(std::istream &I) override { getDecoder(I) >> (x); } #define _SPIRV_DEF_ENCDEC2(x, y) \ void encode(spv_ostream &O) const override { getEncoder(O) << (x) << (y); } \ void decode(std::istream &I) override { getDecoder(I) >> (x) >> (y); } #define _SPIRV_DEF_ENCDEC3(x, y, z) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z); \ } \ void decode(std::istream &I) override { getDecoder(I) >> (x) >> (y) >> (z); } #define _SPIRV_DEF_ENCDEC4(x, y, z, u) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u); \ } #define _SPIRV_DEF_ENCDEC5(x, y, z, u, v) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v); \ } #define _SPIRV_DEF_ENCDEC6(x, y, z, u, v, w) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w); \ } #define _SPIRV_DEF_ENCDEC7(x, y, z, u, v, w, r) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r); \ } #define _SPIRV_DEF_ENCDEC8(x, y, z, u, v, w, r, s) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r) << (s); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r) >> (s); \ } #define _SPIRV_DEF_ENCDEC9(x, y, z, u, v, w, r, s, t) \ void encode(spv_ostream &O) const override { \ getEncoder(O) << (x) << (y) << (z) << (u) << (v) << (w) << (r) << (s) \ << (t); \ } \ void decode(std::istream &I) override { \ getDecoder(I) >> (x) >> (y) >> (z) >> (u) >> (v) >> (w) >> (r) >> (s) >> \ (t); \ } /// All SPIR-V in-memory-representation entities inherits from SPIRVEntry. /// Usually there are two flavors of constructors of SPIRV objects: /// /// 1. complete constructor: It requires all the parameters needed to create a /// SPIRV entity with complete information which can be validated. It is /// usually used by LLVM/SPIR-V translator to create SPIRV object /// corresponding to LLVM object. Such constructor calls validate() at /// the end of the construction. /// /// 2. incomplete constructor: For leaf classes, it has no parameters. /// It is usually called by SPIRVEntry::make(opcode) to create an incomplete /// object which should not be validated. Then setWordCount(count) is /// called to fix the size of the object if it is variable, and then the /// information is filled by the virtual function decode(istream). /// After that the object can be validated. /// /// To add a new SPIRV class: /// /// 1. It is recommended to name the class as SPIRVXXX if it has a fixed op code /// OpXXX. Although it is not mandatory, doing this facilitates adding it to /// the table of the factory function SPIRVEntry::create(). /// 2. Inherit from proper SPIRV class such as SPIRVType, SPIRVValue, /// SPIRVInstruction, etc. /// 3. Implement virtual function encode(), decode(), validate(). /// 4. If the object has variable size, implement virtual function /// setWordCount(). /// 5. If the class has special attributes, e.g. having no id, or having no /// type as a value, set them in the constructors. /// 6. If the class may represent SPIRV entity which has been added in version /// later than 1.0, implement virtual function getRequiredSPIRVVersion(). /// To automaticly update module's version you can also call protected /// function updateModuleVersion() in the constructor. /// 7. Add the class to the Table of SPIRVEntry::create(). /// 8. Add the class to SPIRVToLLVM and LLVMToSPIRV. class SPIRVEntry { public: enum SPIRVEntryAttrib { SPIRVEA_DEFAULT = 0, SPIRVEA_NOID = 1, // Entry has no valid id SPIRVEA_NOTYPE = 2, // Value has no type }; // Complete constructor for objects with id SPIRVEntry(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode, SPIRVId TheId) : Module(M), OpCode(TheOpCode), Id(TheId), Attrib(SPIRVEA_DEFAULT), WordCount(TheWordCount), Line(nullptr) { SPIRVEntry::validate(); } // Complete constructor for objects without id SPIRVEntry(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode) : Module(M), OpCode(TheOpCode), Id(SPIRVID_INVALID), Attrib(SPIRVEA_NOID), WordCount(TheWordCount), Line(nullptr) { SPIRVEntry::validate(); } // Incomplete constructor SPIRVEntry(Op TheOpCode) : Module(NULL), OpCode(TheOpCode), Id(SPIRVID_INVALID), Attrib(SPIRVEA_DEFAULT), WordCount(0), Line(nullptr) {} SPIRVEntry() : Module(NULL), OpCode(OpNop), Id(SPIRVID_INVALID), Attrib(SPIRVEA_DEFAULT), WordCount(0), Line(nullptr) {} virtual ~SPIRVEntry() {} bool exist(SPIRVId) const; template T *get(SPIRVId TheId) const { return static_cast(getEntry(TheId)); } SPIRVEntry *getEntry(SPIRVId) const; SPIRVEntry *getOrCreate(SPIRVId TheId) const; SPIRVValue *getValue(SPIRVId TheId) const; std::vector getValues(const std::vector &) const; std::vector getIds(const std::vector) const; SPIRVType *getValueType(SPIRVId TheId) const; std::vector getValueTypes(const std::vector &) const; virtual SPIRVDecoder getDecoder(std::istream &); virtual SPIRVEncoder getEncoder(spv_ostream &) const; SPIRVErrorLog &getErrorLog() const; SPIRVId getId() const { assert(hasId()); return Id; } std::shared_ptr getLine() const { return Line; } std::shared_ptr getDebugLine() const { return DebugLine; } SPIRVLinkageTypeKind getLinkageType() const; Op getOpCode() const { return OpCode; } SPIRVModule *getModule() const { return Module; } virtual SPIRVCapVec getRequiredCapability() const { return SPIRVCapVec(); } virtual llvm::Optional getRequiredExtension() const { return {}; } const std::string &getName() const { return Name; } size_t getNumDecorations() const { return Decorates.size(); } bool hasDecorate(Decoration Kind, size_t Index = 0, SPIRVWord *Result = 0) const; bool hasDecorateId(Decoration Kind, size_t Index = 0, SPIRVId *Result = 0) const; bool hasMemberDecorate(Decoration Kind, size_t Index = 0, SPIRVWord MemberNumber = 0, SPIRVWord *Result = 0) const; std::vector getDecorationLiterals(Decoration Kind) const; std::vector getDecorationIdLiterals(Decoration Kind) const; std::vector getMemberDecorationLiterals(Decoration Kind, SPIRVWord MemberNumber) const; std::vector getDecorationStringLiteral(Decoration Kind) const; std::vector getMemberDecorationStringLiteral(Decoration Kind, SPIRVWord MemberNumber) const; std::set getDecorate(Decoration Kind, size_t Index = 0) const; std::vector getDecorations(Decoration Kind) const; std::vector getDecorations() const; std::set getDecorateId(Decoration Kind, size_t Index = 0) const; std::vector getDecorationIds(Decoration Kind) const; bool hasId() const { return !(Attrib & SPIRVEA_NOID); } bool hasLine() const { return Line != nullptr; } bool hasLinkageType() const; bool isAtomic() const { return isAtomicOpCode(OpCode); } bool isBasicBlock() const { return isLabel(); } bool isExtInst() const { return OpCode == OpExtInst; } bool isExtInst(const SPIRVExtInstSetKind InstSet) const; bool isExtInst(const SPIRVExtInstSetKind InstSet, const SPIRVWord ExtOp) const; bool isDecorate() const { return OpCode == OpDecorate; } bool isDecorateId() const { return OpCode == OpDecorateId; } bool isMemberDecorate() const { return OpCode == OpMemberDecorate; } bool isForward() const { return OpCode == internal::OpForward; } bool isLabel() const { return OpCode == OpLabel; } bool isUndef() const { return OpCode == OpUndef; } bool isControlBarrier() const { return OpCode == OpControlBarrier; } bool isMemoryBarrier() const { return OpCode == OpMemoryBarrier; } bool isVariable() const { return OpCode == OpVariable; } bool isEndOfBlock() const; virtual bool isInst() const { return false; } virtual bool isOperandLiteral(unsigned Index) const { assert(0 && "not implemented"); return false; } virtual bool isImplemented() const { return true; } void addDecorate(SPIRVDecorate *); void addDecorate(SPIRVDecorateId *); void addDecorate(Decoration Kind); void addDecorate(Decoration Kind, SPIRVWord Literal); void eraseDecorate(Decoration); void eraseDecorateId(Decoration); void addMemberDecorate(SPIRVMemberDecorate *); void addMemberDecorate(SPIRVWord MemberNumber, Decoration Kind); void addMemberDecorate(SPIRVWord MemberNumber, Decoration Kind, SPIRVWord Literal); void eraseMemberDecorate(SPIRVWord MemberNumber, Decoration Kind); void setHasNoId() { Attrib |= SPIRVEA_NOID; } void setId(SPIRVId TheId) { Id = TheId; } void setLine(const std::shared_ptr &L); void setDebugLine(const std::shared_ptr &DL); void setLinkageType(SPIRVLinkageTypeKind); void setModule(SPIRVModule *TheModule); void setName(const std::string &TheName); virtual void setScope(SPIRVEntry *Scope){}; void takeAnnotations(SPIRVForward *); void takeDecorates(SPIRVEntry *); void takeDecorateIds(SPIRVEntry *); void takeMemberDecorates(SPIRVEntry *); void replaceTargetIdInDecorates(SPIRVId); /// After a SPIRV entry is created during reading SPIRV binary by default /// constructor, this function is called to allow the SPIRV entry to resize /// its variable sized member before decoding the remaining words. virtual void setWordCount(SPIRVWord TheWordCount); /// Create an empty SPIRV object by op code, e.g. OpTypeInt creates /// SPIRVTypeInt. static SPIRVEntry *create(Op); static std::unique_ptr createUnique(Op); /// Create an empty extended instruction. static std::unique_ptr createUnique(SPIRVExtInstSetKind Set, unsigned ExtOp); friend spv_ostream &operator<<(spv_ostream &O, const SPIRVEntry &E); friend std::istream &operator>>(std::istream &I, SPIRVEntry &E); virtual void encodeLine(spv_ostream &O) const; virtual void encodeDebugLine(spv_ostream &O) const; virtual void encodeAll(spv_ostream &O) const; virtual void encodeName(spv_ostream &O) const; virtual void encodeChildren(spv_ostream &O) const; virtual void encodeDecorate(spv_ostream &O) const; virtual void encodeWordCountOpCode(spv_ostream &O) const; virtual void encode(spv_ostream &O) const; virtual void decode(std::istream &I); friend class SPIRVDecoder; /// Checks the integrity of the object. virtual void validate() const { assert(Module && "Invalid module"); assert(OpCode != OpNop && "Invalid op code"); assert((!hasId() || isValidId(Id)) && "Invalid Id"); if (WordCount > 65535) { std::stringstream SS; SS << "Id: " << Id << ", OpCode: " << OpCodeNameMap::map(OpCode) << ", Name: \"" << Name << "\"\n"; getErrorLog().checkError(false, SPIRVEC_InvalidWordCount, SS.str()); } } void validateFunctionControlMask(SPIRVWord FCtlMask) const; void validateValues(const std::vector &) const; void validateBuiltin(SPIRVWord, SPIRVWord) const; // By default assume SPIRV 1.0 as required version virtual SPIRVWord getRequiredSPIRVVersion() const { return static_cast(VersionNumber::SPIRV_1_0); } virtual std::vector getNonLiteralOperands() const { return std::vector(); } protected: /// An entry may have multiple FuncParamAttr decorations. typedef std::multimap DecorateMapType; typedef std::multimap DecorateIdMapType; typedef std::map, const SPIRVMemberDecorate *> MemberDecorateMapType; bool canHaveMemberDecorates() const { return OpCode == OpTypeStruct || OpCode == internal::OpForward; } MemberDecorateMapType &getMemberDecorates() { assert(canHaveMemberDecorates()); return MemberDecorates; } void updateModuleVersion() const; SPIRVModule *Module; Op OpCode; SPIRVId Id; std::string Name; unsigned Attrib; SPIRVWord WordCount; DecorateMapType Decorates; DecorateIdMapType DecorateIds; MemberDecorateMapType MemberDecorates; std::shared_ptr Line; std::shared_ptr DebugLine; }; class SPIRVEntryNoIdGeneric : public SPIRVEntry { public: SPIRVEntryNoIdGeneric(SPIRVModule *M, unsigned TheWordCount, Op OC) : SPIRVEntry(M, TheWordCount, OC) { setAttr(); } SPIRVEntryNoIdGeneric(Op OC) : SPIRVEntry(OC) { setAttr(); } protected: void setAttr() { setHasNoId(); } }; template class SPIRVEntryNoId : public SPIRVEntryNoIdGeneric { public: SPIRVEntryNoId(SPIRVModule *M, unsigned TheWordCount) : SPIRVEntryNoIdGeneric(M, TheWordCount, OC) {} SPIRVEntryNoId() : SPIRVEntryNoIdGeneric(OC) {} }; template class SPIRVEntryOpCodeOnly : public SPIRVEntryNoId { public: SPIRVEntryOpCodeOnly() { SPIRVEntry::WordCount = 1; validate(); } protected: _SPIRV_DEF_ENCDEC0 void validate() const override { assert(isValidId(SPIRVEntry::OpCode)); } }; template class SPIRVEntryUnimplemented : public SPIRVEntryNoId { public: SPIRVEntryUnimplemented() { SPIRVEntry::WordCount = 1; validate(); } bool isImplemented() const override { return false; } protected: _SPIRV_DEF_ENCDEC0 void validate() const override { assert(isValidId(SPIRVEntry::OpCode)); } }; class SPIRVAnnotationGeneric : public SPIRVEntryNoIdGeneric { public: // Complete constructor SPIRVAnnotationGeneric(SPIRVModule *TheModule, unsigned TheWordCount, Op OC, SPIRVId TheTarget = SPIRVID_INVALID) : SPIRVEntryNoIdGeneric(TheModule, TheWordCount, OC), Target(TheTarget) {} // Incomplete constructor SPIRVAnnotationGeneric(Op OC) : SPIRVEntryNoIdGeneric(OC), Target(SPIRVID_INVALID) {} SPIRVId getTargetId() const { return Target; } SPIRVForward *getOrCreateTarget() const; void setTargetId(SPIRVId T) { Target = T; } protected: SPIRVId Target; }; class SPIRVAnnotation : public SPIRVAnnotationGeneric { public: // Complete constructor SPIRVAnnotation(Op OC, const SPIRVEntry *TheTarget, unsigned TheWordCount) : SPIRVAnnotationGeneric(TheTarget->getModule(), TheWordCount, OC, TheTarget->getId()) {} // Incomplete constructors SPIRVAnnotation(Op OC) : SPIRVAnnotationGeneric(OC) {} SPIRVAnnotation() : SPIRVAnnotationGeneric(OpNop) {} }; class SPIRVEntryPoint : public SPIRVAnnotation { public: static const SPIRVWord FixedWC = 4; SPIRVEntryPoint(SPIRVModule *TheModule, SPIRVExecutionModelKind, SPIRVId TheId, const std::string &TheName, std::vector Variables); SPIRVEntryPoint() : SPIRVAnnotation(OpEntryPoint) {} _SPIRV_DCL_ENCDEC protected: SPIRVExecutionModelKind ExecModel; std::string Name; private: std::vector Variables; }; class SPIRVName : public SPIRVAnnotation { public: // Complete constructor SPIRVName(const SPIRVEntry *TheTarget, const std::string &TheStr); // Incomplete constructor SPIRVName() : SPIRVAnnotation(OpName) {} protected: _SPIRV_DCL_ENCDEC void validate() const override; std::string Str; }; class SPIRVMemberName : public SPIRVAnnotation { public: static const SPIRVWord FixedWC = 3; // Complete constructor SPIRVMemberName(const SPIRVEntry *TheTarget, SPIRVWord TheMemberNumber, const std::string &TheStr) : SPIRVAnnotation(OpName, TheTarget, FixedWC + getSizeInWords(TheStr)), MemberNumber(TheMemberNumber), Str(TheStr) { validate(); } // Incomplete constructor SPIRVMemberName() : SPIRVAnnotation(OpName), MemberNumber(SPIRVWORD_MAX) {} protected: _SPIRV_DCL_ENCDEC void validate() const override; SPIRVWord MemberNumber; std::string Str; }; class SPIRVString : public SPIRVEntry { static const Op OC = OpString; static const SPIRVWord FixedWC = 2; public: SPIRVString(SPIRVModule *M, SPIRVId TheId, const std::string &TheStr) : SPIRVEntry(M, FixedWC + getSizeInWords(TheStr), OC, TheId), Str(TheStr) {} SPIRVString() : SPIRVEntry(OC) {} _SPIRV_DCL_ENCDEC const std::string &getStr() const { return Str; } protected: std::string Str; }; class SPIRVLine : public SPIRVEntry { public: static const SPIRVWord WC = 4; // Complete constructor SPIRVLine(SPIRVModule *M, SPIRVId TheFileName, SPIRVWord TheLine, SPIRVWord TheColumn) : SPIRVEntry(M, WC, OpLine), FileName(TheFileName), Line(TheLine), Column(TheColumn) { Attrib = SPIRVEA_NOID | SPIRVEA_NOTYPE; validate(); } // Incomplete constructor SPIRVLine() : SPIRVEntry(OpLine), FileName(SPIRVID_INVALID), Line(SPIRVWORD_MAX), Column(SPIRVWORD_MAX) { Attrib = SPIRVEA_NOID | SPIRVEA_NOTYPE; } SPIRVWord getColumn() const { return Column; } void setColumn(const SPIRVWord Column) { this->Column = Column; } SPIRVId getFileName() const { return FileName; } const std::string &getFileNameStr() const { return get(FileName)->getStr(); } void setFileName(const SPIRVId FileName) { this->FileName = FileName; } SPIRVWord getLine() const { return Line; } void setLine(const SPIRVWord Line) { this->Line = Line; } bool operator!=(const SPIRVLine &O) const { return !equals(O.FileName, O.Line, O.Column); } bool equals(const SPIRVId TheFileName, const SPIRVWord TheLine, const SPIRVWord TheColumn) const { return FileName == TheFileName && Line == TheLine && Column == TheColumn; } protected: _SPIRV_DCL_ENCDEC void validate() const override; SPIRVId FileName; SPIRVWord Line; SPIRVWord Column; }; class SPIRVExecutionMode : public SPIRVAnnotation { public: // Complete constructor for LocalSize, LocalSizeHint SPIRVExecutionMode(Op OC, SPIRVEntry *TheTarget, SPIRVExecutionModeKind TheExecMode, SPIRVWord X, SPIRVWord Y, SPIRVWord Z) : SPIRVAnnotation(OC, TheTarget, 6), ExecMode(TheExecMode) { WordLiterals.push_back(X); WordLiterals.push_back(Y); WordLiterals.push_back(Z); updateModuleVersion(); } // Complete constructor for VecTypeHint, SubgroupSize, SubgroupsPerWorkgroup SPIRVExecutionMode(Op OC, SPIRVEntry *TheTarget, SPIRVExecutionModeKind TheExecMode, SPIRVWord Code) : SPIRVAnnotation(OC, TheTarget, 4), ExecMode(TheExecMode) { WordLiterals.push_back(Code); } // Complete constructor for ContractionOff SPIRVExecutionMode(Op OC, SPIRVEntry *TheTarget, SPIRVExecutionModeKind TheExecMode) : SPIRVAnnotation(OC, TheTarget, 3), ExecMode(TheExecMode) { updateModuleVersion(); } // Incomplete constructor SPIRVExecutionMode() : SPIRVAnnotation(OpExecutionMode), ExecMode(ExecutionModeInvocations) {} SPIRVExecutionModeKind getExecutionMode() const { return ExecMode; } const std::vector &getLiterals() const { return WordLiterals; } SPIRVCapVec getRequiredCapability() const override { return getCapability(ExecMode); } SPIRVWord getRequiredSPIRVVersion() const override { switch (ExecMode) { case ExecutionModeFinalizer: case ExecutionModeInitializer: case ExecutionModeSubgroupSize: case ExecutionModeSubgroupsPerWorkgroup: return static_cast(VersionNumber::SPIRV_1_1); default: return static_cast(VersionNumber::SPIRV_1_0); } } llvm::Optional getRequiredExtension() const override { switch (static_cast(ExecMode)) { case ExecutionModeMaximumRegistersINTEL: case ExecutionModeMaximumRegistersIdINTEL: case ExecutionModeNamedMaximumRegistersINTEL: return ExtensionID::SPV_INTEL_maximum_registers; default: return {}; } } protected: _SPIRV_DCL_ENCDEC SPIRVExecutionModeKind ExecMode; std::vector WordLiterals; }; class SPIRVExecutionModeId : public SPIRVExecutionMode { public: // Complete constructor for LocalSizeId, LocalSizeHintId SPIRVExecutionModeId(SPIRVEntry *TheTarget, SPIRVExecutionModeKind TheExecMode, SPIRVWord X, SPIRVWord Y, SPIRVWord Z) : SPIRVExecutionMode(OpExecutionModeId, TheTarget, TheExecMode, X, Y, Z) { updateModuleVersion(); } // Complete constructor for SubgroupsPerWorkgroupId SPIRVExecutionModeId(SPIRVEntry *TheTarget, SPIRVExecutionModeKind TheExecMode, SPIRVWord Code) : SPIRVExecutionMode(OpExecutionModeId, TheTarget, TheExecMode, Code) { updateModuleVersion(); } // Incomplete constructor SPIRVExecutionModeId() : SPIRVExecutionMode() {} SPIRVWord getRequiredSPIRVVersion() const override { return static_cast(VersionNumber::SPIRV_1_2); } }; class SPIRVComponentExecutionModes { typedef std::multimap SPIRVExecutionModeMap; typedef std::pair SPIRVExecutionModeRange; public: void addExecutionMode(SPIRVExecutionMode *ExecMode) { // There should not be more than 1 execution mode kind except the ones // mentioned in SPV_KHR_float_controls and SPV_INTEL_float_controls2. #ifndef NDEBUG auto IsDenorm = [](auto EMK) { return EMK == ExecutionModeDenormPreserve || EMK == ExecutionModeDenormFlushToZero; }; auto IsRoundingMode = [](auto EMK) { return EMK == ExecutionModeRoundingModeRTE || EMK == ExecutionModeRoundingModeRTZ || EMK == ExecutionModeRoundingModeRTPINTEL || EMK == ExecutionModeRoundingModeRTNINTEL; }; auto IsFPMode = [](auto EMK) { return EMK == ExecutionModeFloatingPointModeALTINTEL || EMK == ExecutionModeFloatingPointModeIEEEINTEL; }; auto IsOtherFP = [](auto EMK) { return EMK == ExecutionModeSignedZeroInfNanPreserve; }; auto IsFloatControl = [&](auto EMK) { return IsDenorm(EMK) || IsRoundingMode(EMK) || IsFPMode(EMK) || IsOtherFP(EMK); }; auto IsMaxRegisters = [&](auto EMK) { return EMK == ExecutionModeMaximumRegistersINTEL || EMK == ExecutionModeMaximumRegistersIdINTEL || EMK == ExecutionModeNamedMaximumRegistersINTEL; }; auto IsCompatible = [&](SPIRVExecutionMode *EM0, SPIRVExecutionMode *EM1) { if (EM0->getTargetId() != EM1->getTargetId()) return true; auto EMK0 = EM0->getExecutionMode(); auto EMK1 = EM1->getExecutionMode(); if (!IsFloatControl(EMK0) || !IsFloatControl(EMK1)) return EMK0 != EMK1; auto TW0 = EM0->getLiterals().at(0); auto TW1 = EM1->getLiterals().at(0); if (TW0 != TW1) return true; return !(IsDenorm(EMK0) && IsDenorm(EMK1)) && !(IsRoundingMode(EMK0) && IsRoundingMode(EMK1)) && !(IsFPMode(EMK0) && IsFPMode(EMK1)) && !(IsMaxRegisters(EMK0) && IsMaxRegisters(EMK1)); }; for (auto I = ExecModes.begin(); I != ExecModes.end(); ++I) { assert(IsCompatible(ExecMode, (*I).second) && "Found incompatible execution modes"); } #endif // !NDEBUG SPIRVExecutionModeKind EMK = ExecMode->getExecutionMode(); ExecModes.emplace(EMK, ExecMode); } SPIRVExecutionMode *getExecutionMode(SPIRVExecutionModeKind EMK) const { auto Loc = ExecModes.find(EMK); if (Loc == ExecModes.end()) return nullptr; return Loc->second; } SPIRVExecutionModeId *getExecutionModeId(SPIRVExecutionModeKind EMK) const { auto Loc = ExecModes.find(EMK); if (Loc == ExecModes.end()) return nullptr; return static_cast(Loc->second); } SPIRVExecutionModeRange getExecutionModeRange(SPIRVExecutionModeKind EMK) const { return ExecModes.equal_range(EMK); } protected: SPIRVExecutionModeMap ExecModes; }; class SPIRVExtInstImport : public SPIRVEntry { public: const static Op OC = OpExtInstImport; // Complete constructor SPIRVExtInstImport(SPIRVModule *TheModule, SPIRVId TheId, const std::string &TheStr); // Incomplete constructor SPIRVExtInstImport() : SPIRVEntry(OC) {} protected: _SPIRV_DCL_ENCDEC void validate() const override; std::string Str; }; class SPIRVMemoryModel : public SPIRVEntryNoId { public: SPIRVMemoryModel(SPIRVModule *M) : SPIRVEntryNoId(M, 3) {} SPIRVMemoryModel() {} _SPIRV_DCL_ENCDEC void validate() const override; }; class SPIRVSource : public SPIRVEntryNoId { public: SPIRVSource(SPIRVModule *M) : SPIRVEntryNoId(M, 3) {} SPIRVSource() {} _SPIRV_DCL_ENCDEC }; class SPIRVSourceExtension : public SPIRVEntryNoId { public: SPIRVSourceExtension(SPIRVModule *M, const std::string &SS); SPIRVSourceExtension() {} _SPIRV_DCL_ENCDEC private: std::string S; }; class SPIRVExtension : public SPIRVEntryNoId { public: SPIRVExtension(SPIRVModule *M, const std::string &SS); SPIRVExtension() {} std::string getExtensionName() const { return S; } _SPIRV_DCL_ENCDEC private: std::string S; }; class SPIRVCapability : public SPIRVEntryNoId { public: SPIRVCapability(SPIRVModule *M, SPIRVCapabilityKind K); SPIRVCapability() : Kind(CapabilityMatrix) {} _SPIRV_DCL_ENCDEC SPIRVWord getRequiredSPIRVVersion() const override { switch (Kind) { case CapabilityGroupNonUniform: case CapabilityGroupNonUniformVote: case CapabilityGroupNonUniformArithmetic: case CapabilityGroupNonUniformBallot: case CapabilityGroupNonUniformShuffle: case CapabilityGroupNonUniformShuffleRelative: case CapabilityGroupNonUniformClustered: return static_cast(VersionNumber::SPIRV_1_3); case CapabilityNamedBarrier: case CapabilitySubgroupDispatch: case CapabilityPipeStorage: return static_cast(VersionNumber::SPIRV_1_1); default: return static_cast(VersionNumber::SPIRV_1_0); } } llvm::Optional getRequiredExtension() const override { switch (static_cast(Kind)) { case CapabilityRoundToInfinityINTEL: case CapabilityFloatingPointModeINTEL: case CapabilityFunctionFloatControlINTEL: return ExtensionID::SPV_INTEL_float_controls2; case CapabilityVectorComputeINTEL: case CapabilityVectorAnyINTEL: return ExtensionID::SPV_INTEL_vector_compute; case CapabilityFunctionVariantsINTEL: case CapabilitySpecConditionalINTEL: return ExtensionID::SPV_INTEL_function_variants; case internal::CapabilityBFloat16ArithmeticINTEL: return ExtensionID::SPV_INTEL_bfloat16_arithmetic; default: return {}; } } private: SPIRVCapabilityKind Kind; }; template T *bcast(SPIRVEntry *E) { return static_cast(E); } template bool isa(SPIRVEntry *E) { return E ? E->getOpCode() == OC : false; } template class SPIRVContinuedInstINTELBase : public SPIRVEntryNoId { public: template using EnableIfCompositeConst = typename std::enable_if_t<_OC == OpConstantCompositeContinuedINTEL || _OC == OpSpecConstantCompositeContinuedINTEL, T>; // Complete constructor SPIRVContinuedInstINTELBase(SPIRVModule *M, const std::vector &TheElements) : SPIRVEntryNoId(M, TheElements.size() + 1) { Elements = SPIRVEntry::getIds(TheElements); validate(); } SPIRVContinuedInstINTELBase(SPIRVModule *M, unsigned NumOfElements) : SPIRVEntryNoId(M, NumOfElements + 1) { Elements.resize(NumOfElements, SPIRVID_INVALID); validate(); } // Incomplete constructor SPIRVContinuedInstINTELBase() : SPIRVEntryNoId() {} template EnableIfCompositeConst> getElements() const { return SPIRVEntry::getValues(Elements); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityLongCompositesINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_long_constant_composite; } SPIRVWord getNumElements() const { return Elements.size(); } protected: void validate() const override; void setWordCount(SPIRVWord WordCount) override { SPIRVEntry::setWordCount(WordCount); Elements.resize(WordCount - 1); } _SPIRV_DCL_ENCDEC std::vector Elements; }; class SPIRVTypeStructContinuedINTEL : public SPIRVContinuedInstINTELBase { public: constexpr static Op OC = OpTypeStructContinuedINTEL; // Complete constructor SPIRVTypeStructContinuedINTEL(SPIRVModule *M, unsigned NumOfElements) : SPIRVContinuedInstINTELBase(M, NumOfElements) {} // Incomplete constructor SPIRVTypeStructContinuedINTEL() : SPIRVContinuedInstINTELBase() {} void setElementId(size_t I, SPIRVId Id) { Elements[I] = Id; } SPIRVType *getMemberType(size_t I) const; }; using SPIRVConstantCompositeContinuedINTEL = SPIRVContinuedInstINTELBase; using SPIRVSpecConstantCompositeContinuedINTEL = SPIRVContinuedInstINTELBase; template struct InstToContinued; template <> struct InstToContinued { using Type = SPIRVTypeStructContinuedINTEL *; constexpr static spv::Op OpCode = OpTypeStructContinuedINTEL; }; template <> struct InstToContinued { using Type = SPIRVConstantCompositeContinuedINTEL *; constexpr static spv::Op OpCode = OpConstantCompositeContinuedINTEL; }; template <> struct InstToContinued { using Type = SPIRVSpecConstantCompositeContinuedINTEL *; constexpr static spv::Op OpCode = OpSpecConstantCompositeContinuedINTEL; }; class SPIRVModuleProcessed : public SPIRVEntryNoId { public: SPIRVModuleProcessed(SPIRVModule *M, const std::string &Process) : SPIRVEntryNoId(M, FixedWC + getSizeInWords(Process)), ProcessStr(Process) { updateModuleVersion(); } SPIRVModuleProcessed() { updateModuleVersion(); } _SPIRV_DCL_ENCDEC void validate() const override; SPIRVWord getRequiredSPIRVVersion() const override { return static_cast(VersionNumber::SPIRV_1_1); } std::string getProcessStr(); private: std::string ProcessStr; static const SPIRVWord FixedWC = 1; }; // ToDo: The following typedef's are place holders for SPIRV entity classes // to be implemented. // Each time a new class is implemented, remove the corresponding typedef. // This is also an indication of how much work is left. #define _SPIRV_OP(x) typedef SPIRVEntryUnimplemented SPIRV##x; _SPIRV_OP(Nop) _SPIRV_OP(SourceContinued) _SPIRV_OP(TypeRuntimeArray) _SPIRV_OP(Image) _SPIRV_OP(ImageTexelPointer) _SPIRV_OP(ImageSampleDrefImplicitLod) _SPIRV_OP(ImageSampleDrefExplicitLod) _SPIRV_OP(ImageSampleProjImplicitLod) _SPIRV_OP(ImageSampleProjExplicitLod) _SPIRV_OP(ImageSampleProjDrefImplicitLod) _SPIRV_OP(ImageSampleProjDrefExplicitLod) _SPIRV_OP(ImageFetch) _SPIRV_OP(ImageGather) _SPIRV_OP(ImageDrefGather) _SPIRV_OP(QuantizeToF16) _SPIRV_OP(ArrayLength) _SPIRV_OP(OuterProduct) _SPIRV_OP(DPdx) _SPIRV_OP(DPdy) _SPIRV_OP(Fwidth) _SPIRV_OP(DPdxFine) _SPIRV_OP(DPdyFine) _SPIRV_OP(FwidthFine) _SPIRV_OP(DPdxCoarse) _SPIRV_OP(DPdyCoarse) _SPIRV_OP(FwidthCoarse) _SPIRV_OP(EmitVertex) _SPIRV_OP(EndPrimitive) _SPIRV_OP(EmitStreamVertex) _SPIRV_OP(EndStreamPrimitive) _SPIRV_OP(Kill) _SPIRV_OP(ImageSparseSampleImplicitLod) _SPIRV_OP(ImageSparseSampleExplicitLod) _SPIRV_OP(ImageSparseSampleDrefImplicitLod) _SPIRV_OP(ImageSparseSampleDrefExplicitLod) _SPIRV_OP(ImageSparseSampleProjImplicitLod) _SPIRV_OP(ImageSparseSampleProjExplicitLod) _SPIRV_OP(ImageSparseSampleProjDrefImplicitLod) _SPIRV_OP(ImageSparseSampleProjDrefExplicitLod) _SPIRV_OP(ImageSparseFetch) _SPIRV_OP(ImageSparseGather) _SPIRV_OP(ImageSparseDrefGather) _SPIRV_OP(ImageSparseTexelsResident) _SPIRV_OP(NoLine) _SPIRV_OP(TypeNamedBarrier) _SPIRV_OP(NamedBarrierInitialize) _SPIRV_OP(MemoryNamedBarrier) _SPIRV_OP(GetKernelMaxNumSubgroups) _SPIRV_OP(GetKernelLocalSizeForSubgroupCount) #undef _SPIRV_OP } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVENTRY_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVEnum.h000066400000000000000000000702631506646577400224670ustar00rootroot00000000000000//===- SPIRVEnum.h - SPIR-V enums -------------------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V enums. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVENUM_H #define SPIRV_LIBSPIRV_SPIRVENUM_H #include "LLVMSPIRVOpts.h" #include "SPIRVOpCode.h" #include "spirv/unified1/spirv.hpp" #include "spirv_internal.hpp" #include using namespace spv; namespace SPIRV { // SPIR-V specification p2.2.1. "Instructions": // - SPIR-V Word size is 32 bits. // - an always consumes one word. typedef uint32_t SPIRVWord; typedef uint32_t SPIRVId; #define SPIRVID_MAX ~0U #define SPIRVID_INVALID ~0U #define SPIRVWORD_MAX ~0U static constexpr unsigned SpirvWordSize = static_cast(sizeof(SPIRVWord)); static constexpr unsigned SpirvWordBitWidth = 32; inline bool isValidId(SPIRVId Id) { return Id != SPIRVID_INVALID && Id != 0; } const static unsigned KSpirvMemOrderSemanticMask = 0x1F; enum SPIRVGeneratorKind { SPIRVGEN_KhronosLLVMSPIRVTranslator = 6, SPIRVGEN_KhronosSPIRVAssembler = 7, }; enum SPIRVInstructionSchemaKind { SPIRVISCH_Default, }; enum SPIRVExtInstSetKind { SPIRVEIS_OpenCL, SPIRVEIS_Debug, SPIRVEIS_OpenCL_DebugInfo_100, SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVEIS_NonSemantic_AuxData, SPIRVEIS_Count, }; enum SPIRVSamplerAddressingModeKind { SPIRVSAM_None = 0, SPIRVSAM_ClampEdge = 2, SPIRVSAM_Clamp = 4, SPIRVSAM_Repeat = 6, SPIRVSAM_RepeatMirrored = 8, SPIRVSAM_Invalid = 255, }; enum SPIRVSamplerFilterModeKind { SPIRVSFM_Nearest = 16, SPIRVSFM_Linear = 32, SPIRVSFM_Invalid = 255, }; typedef spv::Capability SPIRVCapabilityKind; typedef spv::ExecutionModel SPIRVExecutionModelKind; typedef spv::ExecutionMode SPIRVExecutionModeKind; typedef spv::AccessQualifier SPIRVAccessQualifierKind; typedef spv::AddressingModel SPIRVAddressingModelKind; typedef spv::LinkageType SPIRVLinkageTypeKind; typedef spv::MemoryModel SPIRVMemoryModelKind; typedef spv::StorageClass SPIRVStorageClassKind; typedef spv::FunctionControlMask SPIRVFunctionControlMaskKind; typedef spv::FPRoundingMode SPIRVFPRoundingModeKind; typedef spv::FunctionParameterAttribute SPIRVFuncParamAttrKind; typedef spv::BuiltIn SPIRVBuiltinVariableKind; typedef spv::MemoryAccessMask SPIRVMemoryAccessKind; typedef spv::GroupOperation SPIRVGroupOperationKind; typedef spv::Dim SPIRVImageDimKind; typedef std::vector SPIRVCapVec; typedef std::set SPIRVExtSet; template <> inline void SPIRVMap::init() { #define _STRINGIFY(X) #X #define STRINGIFY(X) _STRINGIFY(X) #define EXT(X) add(ExtensionID::X, STRINGIFY(X)); #include "LLVMSPIRVExtensions.inc" #undef EXT #undef STRINGIFY #undef _STRINGIFY } template <> inline void SPIRVMap::init() { add(SPIRVEIS_OpenCL, "OpenCL.std"); add(SPIRVEIS_Debug, "SPIRV.debug"); add(SPIRVEIS_OpenCL_DebugInfo_100, "OpenCL.DebugInfo.100"); add(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, "NonSemantic.Shader.DebugInfo.100"); add(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, "NonSemantic.Shader.DebugInfo.200"); add(SPIRVEIS_NonSemantic_AuxData, "NonSemantic.AuxData"); } typedef SPIRVMap SPIRVBuiltinSetNameMap; template SPIRVCapVec getCapability(K Key) { SPIRVCapVec V; SPIRVMap::find(Key, &V); return V; } #define ADD_VEC_INIT(Cap, ...) \ { \ SPIRVCapabilityKind C[] = __VA_ARGS__; \ const SPIRVCapVec V(C, C + sizeof(C) / sizeof(C[0])); \ add(Cap, V); \ } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(CapabilityShader, {CapabilityMatrix}); ADD_VEC_INIT(CapabilityGeometry, {CapabilityShader}); ADD_VEC_INIT(CapabilityTessellation, {CapabilityShader}); ADD_VEC_INIT(CapabilityVector16, {CapabilityKernel}); ADD_VEC_INIT(CapabilityFloat16Buffer, {CapabilityKernel}); ADD_VEC_INIT(CapabilityInt64Atomics, {CapabilityInt64}); ADD_VEC_INIT(CapabilityImageBasic, {CapabilityKernel}); ADD_VEC_INIT(CapabilityImageReadWrite, {CapabilityImageBasic}); ADD_VEC_INIT(CapabilityImageMipmap, {CapabilityImageBasic}); ADD_VEC_INIT(CapabilityPipes, {CapabilityKernel}); ADD_VEC_INIT(CapabilityBlockingPipesINTEL, {CapabilityKernel}); ADD_VEC_INIT(CapabilityDeviceEnqueue, {CapabilityKernel}); ADD_VEC_INIT(CapabilityLiteralSampler, {CapabilityKernel}); ADD_VEC_INIT(CapabilityAtomicStorage, {CapabilityShader}); ADD_VEC_INIT(CapabilityTessellationPointSize, {CapabilityTessellation}); ADD_VEC_INIT(CapabilityGeometryPointSize, {CapabilityGeometry}); ADD_VEC_INIT(CapabilityImageGatherExtended, {CapabilityShader}); ADD_VEC_INIT(CapabilityStorageImageMultisample, {CapabilityShader}); ADD_VEC_INIT(CapabilityUniformBufferArrayDynamicIndexing, {CapabilityShader}); ADD_VEC_INIT(CapabilitySampledImageArrayDynamicIndexing, {CapabilityShader}); ADD_VEC_INIT(CapabilityStorageBufferArrayDynamicIndexing, {CapabilityShader}); ADD_VEC_INIT(CapabilityStorageImageArrayDynamicIndexing, {CapabilityShader}); ADD_VEC_INIT(CapabilityClipDistance, {CapabilityShader}); ADD_VEC_INIT(CapabilityCullDistance, {CapabilityShader}); ADD_VEC_INIT(CapabilityImageCubeArray, {CapabilitySampledCubeArray}); ADD_VEC_INIT(CapabilitySampleRateShading, {CapabilityShader}); ADD_VEC_INIT(CapabilityImageRect, {CapabilitySampledRect}); ADD_VEC_INIT(CapabilitySampledRect, {CapabilityShader}); ADD_VEC_INIT(CapabilityGenericPointer, {CapabilityAddresses}); ADD_VEC_INIT(CapabilityInt8, {CapabilityKernel}); ADD_VEC_INIT(CapabilityInputAttachment, {CapabilityShader}); ADD_VEC_INIT(CapabilitySparseResidency, {CapabilityShader}); ADD_VEC_INIT(CapabilityMinLod, {CapabilityShader}); ADD_VEC_INIT(CapabilityImage1D, {CapabilitySampled1D}); ADD_VEC_INIT(CapabilitySampledCubeArray, {CapabilityShader}); ADD_VEC_INIT(CapabilityImageBuffer, {CapabilitySampledBuffer}); ADD_VEC_INIT(CapabilityImageMSArray, {CapabilityShader}); ADD_VEC_INIT(CapabilityStorageImageExtendedFormats, {CapabilityShader}); ADD_VEC_INIT(CapabilityImageQuery, {CapabilityShader}); ADD_VEC_INIT(CapabilityDerivativeControl, {CapabilityShader}); ADD_VEC_INIT(CapabilityInterpolationFunction, {CapabilityShader}); ADD_VEC_INIT(CapabilityTransformFeedback, {CapabilityShader}); ADD_VEC_INIT(CapabilityGeometryStreams, {CapabilityGeometry}); ADD_VEC_INIT(CapabilityStorageImageReadWithoutFormat, {CapabilityShader}); ADD_VEC_INIT(CapabilityStorageImageWriteWithoutFormat, {CapabilityShader}); ADD_VEC_INIT(CapabilityMultiViewport, {CapabilityGeometry}); ADD_VEC_INIT(CapabilitySubgroupAvcMotionEstimationINTEL, {CapabilityGroups}); ADD_VEC_INIT(CapabilitySubgroupAvcMotionEstimationIntraINTEL, {CapabilitySubgroupAvcMotionEstimationINTEL}); ADD_VEC_INIT(CapabilitySubgroupAvcMotionEstimationChromaINTEL, {CapabilitySubgroupAvcMotionEstimationIntraINTEL}); ADD_VEC_INIT(internal::CapabilityJointMatrixWIInstructionsINTEL, {internal::CapabilityJointMatrixINTEL}); ADD_VEC_INIT(internal::CapabilityCooperativeMatrixPrefetchINTEL, {CapabilityCooperativeMatrixKHR}); ADD_VEC_INIT(internal::CapabilityCooperativeMatrixCheckedInstructionsINTEL, {CapabilityCooperativeMatrixKHR}); ADD_VEC_INIT(CapabilityBFloat16DotProductKHR, {CapabilityBFloat16TypeKHR}); ADD_VEC_INIT(CapabilityBFloat16CooperativeMatrixKHR, {CapabilityBFloat16TypeKHR, CapabilityCooperativeMatrixKHR}); ADD_VEC_INIT(CapabilityInt4CooperativeMatrixINTEL, {CapabilityInt4TypeINTEL, CapabilityCooperativeMatrixKHR}); ADD_VEC_INIT(internal::CapabilityBFloat16ArithmeticINTEL, {CapabilityBFloat16TypeKHR}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(ExecutionModelVertex, {CapabilityShader}); ADD_VEC_INIT(ExecutionModelTessellationControl, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModelTessellationEvaluation, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModelGeometry, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModelFragment, {CapabilityShader}); ADD_VEC_INIT(ExecutionModelGLCompute, {CapabilityShader}); ADD_VEC_INIT(ExecutionModelKernel, {CapabilityKernel}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(ExecutionModeInvocations, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeSpacingEqual, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeSpacingFractionalEven, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeSpacingFractionalOdd, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeVertexOrderCw, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeVertexOrderCcw, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModePixelCenterInteger, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeOriginUpperLeft, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeOriginLowerLeft, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeEarlyFragmentTests, {CapabilityShader}); ADD_VEC_INIT(ExecutionModePointMode, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeXfb, {CapabilityTransformFeedback}); ADD_VEC_INIT(ExecutionModeDepthReplacing, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeDepthGreater, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeDepthLess, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeDepthUnchanged, {CapabilityShader}); ADD_VEC_INIT(ExecutionModeLocalSizeHint, {CapabilityKernel}); ADD_VEC_INIT(ExecutionModeInputPoints, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeInputLines, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeInputLinesAdjacency, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeTriangles, {CapabilityGeometry, CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeInputTrianglesAdjacency, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeQuads, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeIsolines, {CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeOutputVertices, {CapabilityGeometry, CapabilityTessellation}); ADD_VEC_INIT(ExecutionModeOutputPoints, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeOutputLineStrip, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeOutputTriangleStrip, {CapabilityGeometry}); ADD_VEC_INIT(ExecutionModeVecTypeHint, {CapabilityKernel}); ADD_VEC_INIT(ExecutionModeContractionOff, {CapabilityKernel}); ADD_VEC_INIT(ExecutionModeSubgroupSize, {CapabilitySubgroupDispatch}); ADD_VEC_INIT(ExecutionModeDenormPreserve, {CapabilityDenormPreserve}); ADD_VEC_INIT(ExecutionModeDenormFlushToZero, {CapabilityDenormFlushToZero}); ADD_VEC_INIT(ExecutionModeSignedZeroInfNanPreserve, {CapabilitySignedZeroInfNanPreserve}); ADD_VEC_INIT(ExecutionModeRoundingModeRTE, {CapabilityRoundingModeRTE}); ADD_VEC_INIT(ExecutionModeRoundingModeRTZ, {CapabilityRoundingModeRTZ}); ADD_VEC_INIT(ExecutionModeRoundingModeRTPINTEL, {CapabilityRoundToInfinityINTEL}); ADD_VEC_INIT(ExecutionModeRoundingModeRTNINTEL, {CapabilityRoundToInfinityINTEL}); ADD_VEC_INIT(ExecutionModeFloatingPointModeALTINTEL, {CapabilityFloatingPointModeINTEL}); ADD_VEC_INIT(ExecutionModeFloatingPointModeIEEEINTEL, {CapabilityFloatingPointModeINTEL}); ADD_VEC_INIT(ExecutionModeSharedLocalMemorySizeINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(ExecutionModeStreamingInterfaceINTEL, {CapabilityFPGAKernelAttributesINTEL}); ADD_VEC_INIT(ExecutionModeNamedBarrierCountINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(ExecutionModeMaximumRegistersINTEL, {CapabilityRegisterLimitsINTEL}); ADD_VEC_INIT(ExecutionModeMaximumRegistersIdINTEL, {CapabilityRegisterLimitsINTEL}); ADD_VEC_INIT(ExecutionModeNamedMaximumRegistersINTEL, {CapabilityRegisterLimitsINTEL}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(MemoryModelSimple, {CapabilityShader}); ADD_VEC_INIT(MemoryModelGLSL450, {CapabilityShader}); ADD_VEC_INIT(MemoryModelOpenCL, {CapabilityKernel}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(StorageClassUniform, {CapabilityShader}); ADD_VEC_INIT(StorageClassOutput, {CapabilityShader}); ADD_VEC_INIT(StorageClassPrivate, {CapabilityShader, CapabilityVectorComputeINTEL}); ADD_VEC_INIT(StorageClassGeneric, {CapabilityGenericPointer}); ADD_VEC_INIT(StorageClassPushConstant, {CapabilityShader}); ADD_VEC_INIT(StorageClassAtomicCounter, {CapabilityAtomicStorage}); ADD_VEC_INIT(StorageClassDeviceOnlyINTEL, {CapabilityUSMStorageClassesINTEL}); ADD_VEC_INIT(StorageClassHostOnlyINTEL, {CapabilityUSMStorageClassesINTEL}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(Dim1D, {CapabilitySampled1D}); ADD_VEC_INIT(DimCube, {CapabilityShader}); ADD_VEC_INIT(DimRect, {CapabilitySampledRect}); ADD_VEC_INIT(DimBuffer, {CapabilitySampledBuffer}); ADD_VEC_INIT(DimSubpassData, {CapabilityInputAttachment}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(ImageFormatRgba32f, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba16f, {CapabilityShader}); ADD_VEC_INIT(ImageFormatR32f, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba8, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba8Snorm, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRg32f, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg16f, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR11fG11fB10f, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR16f, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRgba16, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRgb10A2, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg16, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg8, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR16, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR8, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRgba16Snorm, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg16Snorm, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg8Snorm, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR16Snorm, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR8Snorm, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRgba32i, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba16i, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba8i, {CapabilityShader}); ADD_VEC_INIT(ImageFormatR32i, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRg32i, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg16i, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg8i, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR16i, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR8i, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRgba32ui, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba16ui, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgba8ui, {CapabilityShader}); ADD_VEC_INIT(ImageFormatR32ui, {CapabilityShader}); ADD_VEC_INIT(ImageFormatRgb10a2ui, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg32ui, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatRg16ui, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR16ui, {CapabilityStorageImageExtendedFormats}); ADD_VEC_INIT(ImageFormatR8ui, {CapabilityStorageImageExtendedFormats}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(ImageOperandsBiasMask, {CapabilityShader}); ADD_VEC_INIT(ImageOperandsOffsetMask, {CapabilityImageGatherExtended}); ADD_VEC_INIT(ImageOperandsMinLodMask, {CapabilityMinLod}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(DecorationRelaxedPrecision, {CapabilityShader}); ADD_VEC_INIT(DecorationSpecId, {CapabilityKernel}); ADD_VEC_INIT(DecorationBlock, {CapabilityShader}); ADD_VEC_INIT(DecorationBufferBlock, {CapabilityShader}); ADD_VEC_INIT(DecorationRowMajor, {CapabilityMatrix}); ADD_VEC_INIT(DecorationColMajor, {CapabilityMatrix}); ADD_VEC_INIT(DecorationArrayStride, {CapabilityShader}); ADD_VEC_INIT(DecorationMatrixStride, {CapabilityMatrix}); ADD_VEC_INIT(DecorationGLSLShared, {CapabilityShader}); ADD_VEC_INIT(DecorationGLSLPacked, {CapabilityShader}); ADD_VEC_INIT(DecorationCPacked, {CapabilityKernel}); ADD_VEC_INIT(DecorationNoPerspective, {CapabilityShader}); ADD_VEC_INIT(DecorationFlat, {CapabilityShader}); ADD_VEC_INIT(DecorationPatch, {CapabilityTessellation}); ADD_VEC_INIT(DecorationCentroid, {CapabilityShader}); ADD_VEC_INIT(DecorationSample, {CapabilitySampleRateShading}); ADD_VEC_INIT(DecorationInvariant, {CapabilityShader}); ADD_VEC_INIT(DecorationConstant, {CapabilityKernel}); ADD_VEC_INIT(DecorationSaturatedConversion, {CapabilityKernel}); ADD_VEC_INIT(DecorationStream, {CapabilityGeometryStreams}); ADD_VEC_INIT(DecorationLocation, {CapabilityShader}); ADD_VEC_INIT(DecorationComponent, {CapabilityShader}); ADD_VEC_INIT(DecorationIndex, {CapabilityShader}); ADD_VEC_INIT(DecorationBinding, {CapabilityShader}); ADD_VEC_INIT(DecorationDescriptorSet, {CapabilityShader}); ADD_VEC_INIT(DecorationOffset, {CapabilityShader}); ADD_VEC_INIT(DecorationXfbBuffer, {CapabilityTransformFeedback}); ADD_VEC_INIT(DecorationXfbStride, {CapabilityTransformFeedback}); ADD_VEC_INIT(DecorationFuncParamAttr, {CapabilityKernel}); ADD_VEC_INIT(DecorationFPRoundingMode, {CapabilityKernel}); ADD_VEC_INIT(DecorationFPFastMathMode, {CapabilityKernel}); ADD_VEC_INIT(DecorationLinkageAttributes, {CapabilityLinkage}); ADD_VEC_INIT(DecorationNoContraction, {CapabilityShader}); ADD_VEC_INIT(DecorationInputAttachmentIndex, {CapabilityInputAttachment}); ADD_VEC_INIT(DecorationAlignment, {CapabilityKernel}); ADD_VEC_INIT(DecorationRegisterINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationMemoryINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationNumbanksINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationBankwidthINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationMaxPrivateCopiesINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationSinglepumpINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationDoublepumpINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationMaxReplicatesINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationSimpleDualPortINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationMergeINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationBankBitsINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationForcePow2DepthINTEL, {CapabilityFPGAMemoryAttributesINTEL}); ADD_VEC_INIT(DecorationReferencedIndirectlyINTEL, {CapabilityIndirectReferencesINTEL}); ADD_VEC_INIT(DecorationIOPipeStorageINTEL, {CapabilityIOPipesINTEL}); ADD_VEC_INIT(DecorationSideEffectsINTEL, {CapabilityAsmINTEL}); ADD_VEC_INIT(DecorationVectorComputeFunctionINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationVectorComputeVariableINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationGlobalVariableOffsetINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationFuncParamIOKindINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationStackCallINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationSIMTCallINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationBurstCoalesceINTEL, {CapabilityFPGAMemoryAccessesINTEL}); ADD_VEC_INIT(DecorationCacheSizeINTEL, {CapabilityFPGAMemoryAccessesINTEL}); ADD_VEC_INIT(DecorationDontStaticallyCoalesceINTEL, {CapabilityFPGAMemoryAccessesINTEL}); ADD_VEC_INIT(DecorationPrefetchINTEL, {CapabilityFPGAMemoryAccessesINTEL}); ADD_VEC_INIT(DecorationBufferLocationINTEL, {CapabilityFPGABufferLocationINTEL}); ADD_VEC_INIT(DecorationFunctionRoundingModeINTEL, {CapabilityFunctionFloatControlINTEL}); ADD_VEC_INIT(DecorationFunctionDenormModeINTEL, {CapabilityFunctionFloatControlINTEL}); ADD_VEC_INIT(DecorationFunctionFloatingPointModeINTEL, {CapabilityFunctionFloatControlINTEL}); ADD_VEC_INIT(DecorationSingleElementVectorINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationAliasScopeINTEL, {CapabilityMemoryAccessAliasingINTEL}); ADD_VEC_INIT(DecorationNoAliasINTEL, {CapabilityMemoryAccessAliasingINTEL}); ADD_VEC_INIT(DecorationMediaBlockIOINTEL, {CapabilityVectorComputeINTEL}); ADD_VEC_INIT(DecorationStallEnableINTEL, {CapabilityFPGAClusterAttributesINTEL}); ADD_VEC_INIT(DecorationFuseLoopsInFunctionINTEL, {CapabilityLoopFuseINTEL}); ADD_VEC_INIT(internal::DecorationMathOpDSPModeINTEL, {internal::CapabilityFPGADSPControlINTEL}); ADD_VEC_INIT(internal::DecorationInitiationIntervalINTEL, {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL}); ADD_VEC_INIT(internal::DecorationMaxConcurrencyINTEL, {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL}); ADD_VEC_INIT(internal::DecorationPipelineEnableINTEL, {internal::CapabilityFPGAInvocationPipeliningAttributesINTEL}); ADD_VEC_INIT(internal::DecorationRuntimeAlignedINTEL, {internal::CapabilityRuntimeAlignedAttributeINTEL}); ADD_VEC_INIT(internal::DecorationHostAccessINTEL, {internal::CapabilityGlobalVariableDecorationsINTEL}); ADD_VEC_INIT(internal::DecorationInitModeINTEL, {internal::CapabilityGlobalVariableDecorationsINTEL}); ADD_VEC_INIT(internal::DecorationImplementInCSRINTEL, {internal::CapabilityGlobalVariableDecorationsINTEL}); ADD_VEC_INIT(DecorationHostAccessINTEL, {CapabilityGlobalVariableHostAccessINTEL}); ADD_VEC_INIT(DecorationInitModeINTEL, {CapabilityGlobalVariableFPGADecorationsINTEL}); ADD_VEC_INIT(DecorationImplementInRegisterMapINTEL, {CapabilityGlobalVariableFPGADecorationsINTEL}); ADD_VEC_INIT(internal::DecorationArgumentAttributeINTEL, {CapabilityFunctionPointersINTEL}); ADD_VEC_INIT(DecorationFPMaxErrorDecorationINTEL, {CapabilityFPMaxErrorINTEL}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(BuiltInPosition, {CapabilityShader}); ADD_VEC_INIT(BuiltInPointSize, {CapabilityShader}); ADD_VEC_INIT(BuiltInClipDistance, {CapabilityClipDistance}); ADD_VEC_INIT(BuiltInCullDistance, {CapabilityCullDistance}); ADD_VEC_INIT(BuiltInVertexId, {CapabilityShader}); ADD_VEC_INIT(BuiltInInstanceId, {CapabilityShader}); ADD_VEC_INIT(BuiltInPrimitiveId, {CapabilityGeometry, CapabilityTessellation}); ADD_VEC_INIT(BuiltInInvocationId, {CapabilityGeometry, CapabilityTessellation}); ADD_VEC_INIT(BuiltInLayer, {CapabilityGeometry}); ADD_VEC_INIT(BuiltInViewportIndex, {CapabilityMultiViewport}); ADD_VEC_INIT(BuiltInTessLevelOuter, {CapabilityTessellation}); ADD_VEC_INIT(BuiltInTessLevelInner, {CapabilityTessellation}); ADD_VEC_INIT(BuiltInTessCoord, {CapabilityTessellation}); ADD_VEC_INIT(BuiltInPatchVertices, {CapabilityTessellation}); ADD_VEC_INIT(BuiltInFragCoord, {CapabilityShader}); ADD_VEC_INIT(BuiltInPointCoord, {CapabilityShader}); ADD_VEC_INIT(BuiltInFrontFacing, {CapabilityShader}); ADD_VEC_INIT(BuiltInSampleId, {CapabilitySampleRateShading}); ADD_VEC_INIT(BuiltInSamplePosition, {CapabilitySampleRateShading}); ADD_VEC_INIT(BuiltInSampleMask, {CapabilitySampleRateShading}); ADD_VEC_INIT(BuiltInFragDepth, {CapabilityShader}); ADD_VEC_INIT(BuiltInHelperInvocation, {CapabilityShader}); ADD_VEC_INIT(BuiltInWorkDim, {CapabilityKernel}); ADD_VEC_INIT(BuiltInGlobalSize, {CapabilityKernel}); ADD_VEC_INIT(BuiltInEnqueuedWorkgroupSize, {CapabilityKernel}); ADD_VEC_INIT(BuiltInGlobalOffset, {CapabilityKernel}); ADD_VEC_INIT(BuiltInGlobalLinearId, {CapabilityKernel}); ADD_VEC_INIT(BuiltInSubgroupSize, {CapabilityKernel}); ADD_VEC_INIT(BuiltInSubgroupMaxSize, {CapabilityKernel}); ADD_VEC_INIT(BuiltInNumSubgroups, {CapabilityKernel}); ADD_VEC_INIT(BuiltInNumEnqueuedSubgroups, {CapabilityKernel}); ADD_VEC_INIT(BuiltInSubgroupId, {CapabilityKernel}); ADD_VEC_INIT(BuiltInSubgroupLocalInvocationId, {CapabilityKernel}); ADD_VEC_INIT(BuiltInSubgroupEqMask, {CapabilityGroupNonUniformBallot}); ADD_VEC_INIT(BuiltInSubgroupGeMask, {CapabilityGroupNonUniformBallot}); ADD_VEC_INIT(BuiltInSubgroupGtMask, {CapabilityGroupNonUniformBallot}); ADD_VEC_INIT(BuiltInSubgroupLeMask, {CapabilityGroupNonUniformBallot}); ADD_VEC_INIT(BuiltInSubgroupLtMask, {CapabilityGroupNonUniformBallot}); ADD_VEC_INIT(BuiltInVertexIndex, {CapabilityShader}); ADD_VEC_INIT(BuiltInInstanceIndex, {CapabilityShader}); ADD_VEC_INIT(internal::BuiltInSubDeviceIDINTEL, {internal::CapabilityHWThreadQueryINTEL}); ADD_VEC_INIT(internal::BuiltInGlobalHWThreadIDINTEL, {internal::CapabilityHWThreadQueryINTEL}); } template <> inline void SPIRVMap::init() { ADD_VEC_INIT(MemorySemanticsUniformMemoryMask, {CapabilityShader}); ADD_VEC_INIT(MemorySemanticsAtomicCounterMemoryMask, {CapabilityAtomicStorage}); } #undef ADD_VEC_INIT inline unsigned getImageDimension(SPIRVImageDimKind K) { switch (K) { case Dim1D: return 1; case Dim2D: return 2; case Dim3D: return 3; case DimCube: return 2; case DimRect: return 2; case DimBuffer: return 1; default: return 0; } } /// Extract memory order part of SPIR-V memory semantics. inline unsigned extractSPIRVMemOrderSemantic(unsigned Sema) { return Sema & KSpirvMemOrderSemanticMask; } } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVENUM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVError.h000066400000000000000000000161271506646577400226530ustar00rootroot00000000000000//===- SPIRVError.h - SPIR-V error code and checking ------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// // // This file defines SPIRV error code and checking utility. // //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVERROR_H #define SPIRV_LIBSPIRV_SPIRVERROR_H #include "SPIRVDebug.h" #include "SPIRVUtil.h" #include "llvm/IR/Instruction.h" #include #include #include namespace SPIRV { // Check condition and set error code and error msg. // To use this macro, function checkError must be defined in the scope. // Emit absolute path only in debug mode. #ifdef NDEBUG #define SPIRVCK(Condition, ErrCode, ErrMsg) \ getErrorLog().checkError(Condition, SPIRVEC_##ErrCode, \ std::string() + (ErrMsg), #Condition) #else #define SPIRVCK(Condition, ErrCode, ErrMsg) \ getErrorLog().checkError(Condition, SPIRVEC_##ErrCode, \ std::string() + (ErrMsg), #Condition, __FILE__, \ __LINE__) #endif // NDEBUG // Check condition and set error code and error msg. If fail returns false. // Emit absolute path only in debug mode. #ifdef NDEBUG #define SPIRVCKRT(Condition, ErrCode, ErrMsg) \ if (!getErrorLog().checkError(Condition, SPIRVEC_##ErrCode, \ std::string() + (ErrMsg), #Condition)) \ return false; #else #define SPIRVCKRT(Condition, ErrCode, ErrMsg) \ if (!getErrorLog().checkError(Condition, SPIRVEC_##ErrCode, \ std::string() + (ErrMsg), #Condition, \ __FILE__, __LINE__)) \ return false; #endif // NDEBUG // Defines error code enum type SPIRVErrorCode. enum SPIRVErrorCode { #define _SPIRV_OP(x, y) SPIRVEC_##x, #include "SPIRVErrorEnum.h" #undef _SPIRV_OP }; // Defines SPIRVErrorMap which maps error code to a string describing the error. template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) add(SPIRVEC_##x, std::string(#x) + ": " + (y)); #include "SPIRVErrorEnum.h" #undef _SPIRV_OP } typedef SPIRVMap SPIRVErrorMap; class SPIRVErrorLog { public: SPIRVErrorLog() : ErrorCode(SPIRVEC_Success) {} SPIRVErrorCode getError(std::string &ErrMsg) { ErrMsg = ErrorMsg; return ErrorCode; } void setError(SPIRVErrorCode ErrCode, const std::string &ErrMsg) { ErrorCode = ErrCode; ErrorMsg = ErrMsg; } // Check if Condition is satisfied and set ErrCode and DetailedMsg // if not. Returns true if no error. bool checkError(bool Condition, SPIRVErrorCode ErrCode, const std::string &DetailedMsg = "", const char *CondString = nullptr, const char *FileName = nullptr, unsigned LineNumber = 0); // Check if Condition is satisfied and set ErrCode and DetailedMsg with Value // text representation if not. Returns true if no error. bool checkError(bool Condition, SPIRVErrorCode ErrCode, llvm::Value *Value, const std::string &DetailedMsg = "", const char *CondString = nullptr, const char *FileName = nullptr, unsigned LineNumber = 0); protected: SPIRVErrorCode ErrorCode; std::string ErrorMsg; }; inline bool SPIRVErrorLog::checkError(bool Cond, SPIRVErrorCode ErrCode, llvm::Value *Value, const std::string &Msg, const char *CondString, const char *FileName, unsigned LineNo) { // Do early exit to avoid expensive toString() function call unless it is // actually needed. That speeds up translator's execution. if (Cond) return Cond; // Do not overwrite previous failure. if (ErrorCode != SPIRVEC_Success) return Cond; std::string ValueIR = toString(Value); return checkError(Cond, ErrCode, Msg + "\n" + ValueIR, CondString, FileName, LineNo); } inline bool SPIRVErrorLog::checkError(bool Cond, SPIRVErrorCode ErrCode, const std::string &Msg, const char *CondString, const char *FileName, unsigned LineNo) { std::stringstream SS; if (Cond) return Cond; // Do not overwrite previous failure. if (ErrorCode != SPIRVEC_Success) return Cond; SS << SPIRVErrorMap::map(ErrCode) << " " << Msg; if (SPIRVDbgErrorMsgIncludesSourceInfo && FileName) SS << " [Src: " << FileName << ":" << LineNo << " " << CondString << " ]"; setError(ErrCode, SS.str()); switch (SPIRVDbgError) { case SPIRVDbgErrorHandlingKinds::Abort: std::cerr << SS.str() << std::endl; abort(); break; case SPIRVDbgErrorHandlingKinds::Exit: std::cerr << SS.str() << std::endl; std::exit(ErrCode); break; case SPIRVDbgErrorHandlingKinds::Ignore: // Still print info about the error into debug output stream spvdbgs() << SS.str() << '\n'; spvdbgs().flush(); break; } return Cond; } } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVERROR_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVErrorEnum.h000066400000000000000000000037021506646577400234730ustar00rootroot00000000000000/* The error code name should be meaningful since it is part of error message */ _SPIRV_OP(Success, "") _SPIRV_OP(InvalidTargetTriple, "Expects spir-unknown-unknown or spir64-unknown-unknown.") _SPIRV_OP(InvalidSubArch, "Expecting v1.0-v1.6.") _SPIRV_OP(TripleMaxVersionIncompatible, "Triple version and maximum version are incompatible.") _SPIRV_OP(InvalidAddressingModel, "Expects 0-2.") _SPIRV_OP(InvalidMemoryModel, "Expects 0-3.") _SPIRV_OP(InvalidFunctionControlMask, "") _SPIRV_OP(InvalidBuiltinSetName, "Expects OpenCL.std.") _SPIRV_OP(InvalidFunctionCall, "Unexpected llvm intrinsic:\n") _SPIRV_OP(InvalidArraySize, "Array size must be at least 1:") _SPIRV_OP(InvalidBitWidth, "Invalid bit width in input:") _SPIRV_OP(InvalidModule, "Invalid SPIR-V module:") _SPIRV_OP(InvalidLlvmModule, "Invalid LLVM module:") _SPIRV_OP(UnimplementedOpCode, "Unimplemented opcode") _SPIRV_OP(FunctionPointers, "Can't translate function pointer:\n") _SPIRV_OP(InvalidInstruction, "Can't translate llvm instruction:\n") _SPIRV_OP(InvalidWordCount, "Can't encode instruction with word count greater than 65535:\n") _SPIRV_OP(Requires1_1, "Feature requires SPIR-V 1.1 or greater:") _SPIRV_OP(RequiresVersion, "Cannot fulfill SPIR-V version restriction:\n") _SPIRV_OP(RequiresExtension, "Feature requires the following SPIR-V extension:\n") _SPIRV_OP(InvalidMagicNumber, "Invalid Magic Number.") _SPIRV_OP(InvalidVersionNumber, "Invalid Version Number.") _SPIRV_OP(UnspecifiedMemoryModel, "Unspecified Memory Model.") _SPIRV_OP(RepeatedMemoryModel, "Expects a single OpMemoryModel instruction.") _SPIRV_OP(DeprecatedExtension, "Feature requires the following deprecated SPIR-V extension:\n") _SPIRV_OP(InvalidNumberOfOperands, "Number of operands does not match the expected count.") /* This is the last error code to have a maximum valid value to compare to */ _SPIRV_OP(InternalMaxErrorCode, "Unknown error code") SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVExtInst.h000066400000000000000000000270211506646577400231530ustar00rootroot00000000000000//===- SPIRVBuiltin.h - SPIR-V extended instruction -------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V extended instructions. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVEXTINST_H #define SPIRV_LIBSPIRV_SPIRVEXTINST_H #include "NonSemantic.AuxData.h" #include "OpenCL.std.h" #include "SPIRV.debug.h" #include "SPIRVEnum.h" #include "SPIRVUtil.h" #include #include namespace SPIRV { typedef OpenCLLIB::Entrypoints OCLExtOpKind; template <> inline void SPIRVMap::init() { add(OpenCLLIB::Acos, "acos"); add(OpenCLLIB::Acosh, "acosh"); add(OpenCLLIB::Acospi, "acospi"); add(OpenCLLIB::Asin, "asin"); add(OpenCLLIB::Asinh, "asinh"); add(OpenCLLIB::Asinpi, "asinpi"); add(OpenCLLIB::Atan, "atan"); add(OpenCLLIB::Atan2, "atan2"); add(OpenCLLIB::Atanh, "atanh"); add(OpenCLLIB::Atanpi, "atanpi"); add(OpenCLLIB::Atan2pi, "atan2pi"); add(OpenCLLIB::Cbrt, "cbrt"); add(OpenCLLIB::Ceil, "ceil"); add(OpenCLLIB::Copysign, "copysign"); add(OpenCLLIB::Cos, "cos"); add(OpenCLLIB::Cosh, "cosh"); add(OpenCLLIB::Cospi, "cospi"); add(OpenCLLIB::Erfc, "erfc"); add(OpenCLLIB::Erf, "erf"); add(OpenCLLIB::Exp, "exp"); add(OpenCLLIB::Exp2, "exp2"); add(OpenCLLIB::Exp10, "exp10"); add(OpenCLLIB::Expm1, "expm1"); add(OpenCLLIB::Fabs, "fabs"); add(OpenCLLIB::Fdim, "fdim"); add(OpenCLLIB::Floor, "floor"); add(OpenCLLIB::Fma, "fma"); add(OpenCLLIB::Fmax, "fmax"); add(OpenCLLIB::Fmin, "fmin"); add(OpenCLLIB::Fmod, "fmod"); add(OpenCLLIB::Fract, "fract"); add(OpenCLLIB::Frexp, "frexp"); add(OpenCLLIB::Hypot, "hypot"); add(OpenCLLIB::Ilogb, "ilogb"); add(OpenCLLIB::Ldexp, "ldexp"); add(OpenCLLIB::Lgamma, "lgamma"); add(OpenCLLIB::Lgamma_r, "lgamma_r"); add(OpenCLLIB::Log, "log"); add(OpenCLLIB::Log2, "log2"); add(OpenCLLIB::Log10, "log10"); add(OpenCLLIB::Log1p, "log1p"); add(OpenCLLIB::Logb, "logb"); add(OpenCLLIB::Mad, "mad"); add(OpenCLLIB::Maxmag, "maxmag"); add(OpenCLLIB::Minmag, "minmag"); add(OpenCLLIB::Modf, "modf"); add(OpenCLLIB::Nan, "nan"); add(OpenCLLIB::Nextafter, "nextafter"); add(OpenCLLIB::Pow, "pow"); add(OpenCLLIB::Pown, "pown"); add(OpenCLLIB::Powr, "powr"); add(OpenCLLIB::Remainder, "remainder"); add(OpenCLLIB::Remquo, "remquo"); add(OpenCLLIB::Rint, "rint"); add(OpenCLLIB::Rootn, "rootn"); add(OpenCLLIB::Round, "round"); add(OpenCLLIB::Rsqrt, "rsqrt"); add(OpenCLLIB::Sin, "sin"); add(OpenCLLIB::Sincos, "sincos"); add(OpenCLLIB::Sinh, "sinh"); add(OpenCLLIB::Sinpi, "sinpi"); add(OpenCLLIB::Sqrt, "sqrt"); add(OpenCLLIB::Tan, "tan"); add(OpenCLLIB::Tanh, "tanh"); add(OpenCLLIB::Tanpi, "tanpi"); add(OpenCLLIB::Tgamma, "tgamma"); add(OpenCLLIB::Trunc, "trunc"); add(OpenCLLIB::Half_cos, "half_cos"); add(OpenCLLIB::Half_divide, "half_divide"); add(OpenCLLIB::Half_exp, "half_exp"); add(OpenCLLIB::Half_exp2, "half_exp2"); add(OpenCLLIB::Half_exp10, "half_exp10"); add(OpenCLLIB::Half_log, "half_log"); add(OpenCLLIB::Half_log2, "half_log2"); add(OpenCLLIB::Half_log10, "half_log10"); add(OpenCLLIB::Half_powr, "half_powr"); add(OpenCLLIB::Half_recip, "half_recip"); add(OpenCLLIB::Half_rsqrt, "half_rsqrt"); add(OpenCLLIB::Half_sin, "half_sin"); add(OpenCLLIB::Half_sqrt, "half_sqrt"); add(OpenCLLIB::Half_tan, "half_tan"); add(OpenCLLIB::Native_cos, "native_cos"); add(OpenCLLIB::Native_divide, "native_divide"); add(OpenCLLIB::Native_exp, "native_exp"); add(OpenCLLIB::Native_exp2, "native_exp2"); add(OpenCLLIB::Native_exp10, "native_exp10"); add(OpenCLLIB::Native_log, "native_log"); add(OpenCLLIB::Native_log2, "native_log2"); add(OpenCLLIB::Native_log10, "native_log10"); add(OpenCLLIB::Native_powr, "native_powr"); add(OpenCLLIB::Native_recip, "native_recip"); add(OpenCLLIB::Native_rsqrt, "native_rsqrt"); add(OpenCLLIB::Native_sin, "native_sin"); add(OpenCLLIB::Native_sqrt, "native_sqrt"); add(OpenCLLIB::Native_tan, "native_tan"); add(OpenCLLIB::FClamp, "fclamp"); add(OpenCLLIB::Degrees, "degrees"); add(OpenCLLIB::Mix, "mix"); add(OpenCLLIB::FMax_common, "fmax_common"); add(OpenCLLIB::FMin_common, "fmin_common"); add(OpenCLLIB::Radians, "radians"); add(OpenCLLIB::Step, "step"); add(OpenCLLIB::Smoothstep, "smoothstep"); add(OpenCLLIB::Sign, "sign"); add(OpenCLLIB::Cross, "cross"); add(OpenCLLIB::Distance, "distance"); add(OpenCLLIB::Length, "length"); add(OpenCLLIB::Normalize, "normalize"); add(OpenCLLIB::Fast_distance, "fast_distance"); add(OpenCLLIB::Fast_length, "fast_length"); add(OpenCLLIB::Fast_normalize, "fast_normalize"); add(OpenCLLIB::SAbs, "s_abs"); add(OpenCLLIB::SAbs_diff, "s_abs_diff"); add(OpenCLLIB::SAdd_sat, "s_add_sat"); add(OpenCLLIB::UAdd_sat, "u_add_sat"); add(OpenCLLIB::SHadd, "s_hadd"); add(OpenCLLIB::UHadd, "u_hadd"); add(OpenCLLIB::SRhadd, "s_rhadd"); add(OpenCLLIB::URhadd, "u_rhadd"); add(OpenCLLIB::SClamp, "s_clamp"); add(OpenCLLIB::UClamp, "u_clamp"); add(OpenCLLIB::Clz, "clz"); add(OpenCLLIB::Ctz, "ctz"); add(OpenCLLIB::SMad_hi, "s_mad_hi"); add(OpenCLLIB::SMad_sat, "s_mad_sat"); add(OpenCLLIB::UMad_sat, "u_mad_sat"); add(OpenCLLIB::SMax, "s_max"); add(OpenCLLIB::SMin, "s_min"); add(OpenCLLIB::UMax, "u_max"); add(OpenCLLIB::UMin, "u_min"); add(OpenCLLIB::SMul_hi, "s_mul_hi"); add(OpenCLLIB::Rotate, "rotate"); add(OpenCLLIB::SSub_sat, "s_sub_sat"); add(OpenCLLIB::USub_sat, "u_sub_sat"); add(OpenCLLIB::U_Upsample, "u_upsample"); add(OpenCLLIB::S_Upsample, "s_upsample"); add(OpenCLLIB::Popcount, "popcount"); add(OpenCLLIB::SMad24, "s_mad24"); add(OpenCLLIB::UMad24, "u_mad24"); add(OpenCLLIB::SMul24, "s_mul24"); add(OpenCLLIB::UMul24, "u_mul24"); add(OpenCLLIB::Vloadn, "vloadn"); add(OpenCLLIB::Vstoren, "vstoren"); add(OpenCLLIB::Vload_half, "vload_half"); add(OpenCLLIB::Vload_halfn, "vload_halfn"); add(OpenCLLIB::Vstore_half, "vstore_half"); add(OpenCLLIB::Vstore_half_r, "vstore_half_r"); add(OpenCLLIB::Vstore_halfn, "vstore_halfn"); add(OpenCLLIB::Vstore_halfn_r, "vstore_halfn_r"); add(OpenCLLIB::Vloada_halfn, "vloada_halfn"); add(OpenCLLIB::Vstorea_halfn, "vstorea_halfn"); add(OpenCLLIB::Vstorea_halfn_r, "vstorea_halfn_r"); add(OpenCLLIB::Shuffle, "shuffle"); add(OpenCLLIB::Shuffle2, "shuffle2"); add(OpenCLLIB::Printf, "printf"); add(OpenCLLIB::Prefetch, "prefetch"); add(OpenCLLIB::Bitselect, "bitselect"); add(OpenCLLIB::Select, "select"); add(OpenCLLIB::UAbs, "u_abs"); add(OpenCLLIB::UAbs_diff, "u_abs_diff"); add(OpenCLLIB::UMul_hi, "u_mul_hi"); add(OpenCLLIB::UMad_hi, "u_mad_hi"); } SPIRV_DEF_NAMEMAP(OCLExtOpKind, OCLExtOpMap) typedef SPIRVDebug::Instruction SPIRVDebugExtOpKind; template <> inline void SPIRVMap::init() { add(SPIRVDebug::DebugInfoNone, "DebugInfoNone"); add(SPIRVDebug::CompilationUnit, "DebugCompilationUnit"); add(SPIRVDebug::Source, "DebugSource"); add(SPIRVDebug::TypeBasic, "DebugTypeBasic"); add(SPIRVDebug::TypePointer, "DebugTypePointer"); add(SPIRVDebug::TypeArray, "DebugTypeArray"); add(SPIRVDebug::TypeArrayDynamic, "DebugTypeArrayDynamic"); add(SPIRVDebug::TypeVector, "DebugTypeVector"); add(SPIRVDebug::TypeQualifier, "DebugTypeQualifier"); add(SPIRVDebug::TypeFunction, "DebugTypeFunction"); add(SPIRVDebug::TypeComposite, "DebugTypeComposite"); add(SPIRVDebug::TypeMember, "DebugTypeMember"); add(SPIRVDebug::TypeEnum, "DebugTypeEnum"); add(SPIRVDebug::Typedef, "DebugTypedef"); add(SPIRVDebug::TypeTemplateParameter, "DebugTypeTemplateParameter"); add(SPIRVDebug::TypeTemplateParameterPack, "DebugTypeTemplateParameterPack"); add(SPIRVDebug::TypeTemplateTemplateParameter, "DebugTypeTemplateTemplateParameter"); add(SPIRVDebug::TypeTemplate, "DebugTypeTemplate"); add(SPIRVDebug::TypePtrToMember, "DebugTypePtrToMember,"); add(SPIRVDebug::TypeSubrange, "DebugTypeSubrange"); add(SPIRVDebug::TypeString, "DebugTypeString"); add(SPIRVDebug::TypeInheritance, "DebugTypeInheritance"); add(SPIRVDebug::Function, "DebugFunction"); add(SPIRVDebug::FunctionDeclaration, "DebugFunctionDeclaration"); add(SPIRVDebug::LexicalBlock, "DebugLexicalBlock"); add(SPIRVDebug::LexicalBlockDiscriminator, "DebugLexicalBlockDiscriminator"); add(SPIRVDebug::LocalVariable, "DebugLocalVariable"); add(SPIRVDebug::InlinedVariable, "DebugInlinedVariable"); add(SPIRVDebug::GlobalVariable, "DebugGlobalVariable"); add(SPIRVDebug::Declare, "DebugDeclare"); add(SPIRVDebug::Value, "DebugValue"); add(SPIRVDebug::Scope, "DebugScope"); add(SPIRVDebug::NoScope, "DebugNoScope"); add(SPIRVDebug::InlinedAt, "DebugInlinedAt"); add(SPIRVDebug::ImportedEntity, "DebugImportedEntity"); add(SPIRVDebug::ModuleINTEL, "DebugModuleINTEL"); add(SPIRVDebug::Module, "DebugModule"); add(SPIRVDebug::Expression, "DebugExpression"); add(SPIRVDebug::Operation, "DebugOperation"); add(SPIRVDebug::FunctionDefinition, "DebugFunctionDefinition"); add(SPIRVDebug::SourceContinued, "DebugSourceContinued"); add(SPIRVDebug::DebugLine, "DebugLine"); add(SPIRVDebug::DebugNoLine, "DebugNoLine"); add(SPIRVDebug::EntryPoint, "DebugEntryPoint"); add(SPIRVDebug::BuildIdentifier, "DebugBuildIdentifier"); add(SPIRVDebug::StoragePath, "DebugStoragePath"); } SPIRV_DEF_NAMEMAP(SPIRVDebugExtOpKind, SPIRVDebugExtOpMap) typedef NonSemanticAuxData::Instruction NonSemanticAuxDataOpKind; template <> inline void SPIRVMap::init() { add(NonSemanticAuxData::FunctionMetadata, "NonSemanticAuxDataFunctionMetadata"); add(NonSemanticAuxData::FunctionAttribute, "NonSemanticAuxDataFunctionAttribute"); } SPIRV_DEF_NAMEMAP(NonSemanticAuxDataOpKind, NonSemanticAuxDataOpMap) } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVEXTINST_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVFnVar.cpp000066400000000000000000000310661506646577400231300ustar00rootroot00000000000000//===- SPIRVFnVar.cpp -===// // // The LLVM/SPIRV Translator // // Copyright (c) 2025 The Khronos Group Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements functions declared in its header file with the help of /// additional helper functions. /// //===----------------------------------------------------------------------===// #include "SPIRVFnVar.h" using namespace SPIRV; namespace { // Replace SPIRV value with OpConstantTrue/False based on a boolean value. void replaceWithBoolConst(SPIRVModule *BM, SPIRVValue *&OldVal, bool Val) { auto *NewVal = Val ? static_cast( new SPIRVConstantTrue(BM, OldVal->getType(), OldVal->getId())) : static_cast( new SPIRVConstantFalse(BM, OldVal->getType(), OldVal->getId())); [[maybe_unused]] const bool IsSuccess = BM->eraseValue(OldVal); assert(IsSuccess); OldVal = BM->addConstant(NewVal); } // Evaluate a constant pointed to by 'Id' and store the result in 'Res'. // Evaluated instruction is replaced with OpConstantTrue/False depending on the // result. // // To keep this simple, only boolean constants and a subset of OpSpecConstantOp // operations are allowed. Support for more can be implemented as necessary. bool evaluateConstant(SPIRVModule *BM, SPIRVId Id, bool &Res, std::string &ErrMsg) { auto *BV = BM->getValue(Id); const Op OpCode = BV->getOpCode(); assert(isConstantOpCode(OpCode)); assert(BV->getType()->getOpCode() == spv::OpTypeBool); SPIRVWord SpecId = 0; if (BV->hasDecorate(DecorationSpecId, 0, &SpecId)) { if (OpCode != OpSpecConstantTrue && OpCode != OpSpecConstantFalse && OpCode != OpSpecConstantArchitectureINTEL && OpCode != OpSpecConstantTargetINTEL && OpCode != OpSpecConstantCapabilitiesINTEL) { ErrMsg = "Setting only boolean spec constants is supported"; return false; } bool IsTrue = OpCode == OpSpecConstantTrue; uint64_t ConstValue = 0; if (BM->getSpecializationConstant(SpecId, ConstValue)) { IsTrue = ConstValue; } Res = IsTrue; replaceWithBoolConst(BM, BV, Res); return true; } switch (OpCode) { case OpConstantTrue: { Res = true; break; } case OpConstantFalse: { Res = false; break; } case OpSpecConstantTrue: { Res = true; replaceWithBoolConst(BM, BV, Res); break; } case OpSpecConstantFalse: { Res = false; replaceWithBoolConst(BM, BV, Res); break; } case OpSpecConstantArchitectureINTEL: { Res = static_cast(BV)->matchesDevice(); replaceWithBoolConst(BM, BV, Res); break; } case OpSpecConstantTargetINTEL: { Res = static_cast(BV)->matchesDevice(); replaceWithBoolConst(BM, BV, Res); break; } case OpSpecConstantCapabilitiesINTEL: { Res = static_cast(BV)->matchesDevice(); replaceWithBoolConst(BM, BV, Res); break; } case OpSpecConstantOp: { auto OpWords = static_cast(BV)->getOpWords(); auto OpOpCode = static_cast(OpWords[0]); if (OpOpCode != OpLogicalOr && OpOpCode != OpLogicalAnd && OpOpCode != OpLogicalNot) { ErrMsg = "Unsupported operation: Only OpLogicalOr/And/Not are allowed."; return false; } bool Val1 = false; if (!evaluateConstant(BM, OpWords[1], Val1, ErrMsg)) { return false; } if (OpOpCode == OpLogicalNot) { assert(OpWords.size() == 2); if (OpOpCode == OpLogicalNot) { Res = !Val1; } } else { assert(OpWords.size() == 3); bool Val2 = false; if (!evaluateConstant(BM, OpWords[2], Val2, ErrMsg)) { return false; } if (OpOpCode == OpLogicalOr) { Res = Val1 || Val2; } else if (OpOpCode == OpLogicalAnd) { Res = Val1 && Val2; } } replaceWithBoolConst(BM, BV, Res); break; } default: { std::ostringstream S; S << "Evaluating unsupported instruction, opcode: " << OpCode; ErrMsg = S.str(); return false; } } return true; } } // anonymous namespace namespace SPIRV { bool specializeFnVariants(SPIRVModule *BM, std::string &ErrMsg) { // Specialize conditional capabilities std::vector, bool>> CondCapabilities; for (const auto &CondCap : BM->getConditionalCapabilities()) { const SPIRVId Condition = CondCap.first.first; const Capability Cap = CondCap.first.second; const SPIRVConditionalCapabilityINTEL *Entry = CondCap.second; bool ShouldKeep = false; if (!evaluateConstant(BM, Entry->getCondition(), ShouldKeep, ErrMsg)) { return false; } CondCapabilities.emplace_back( std::make_pair(std::make_pair(Condition, Cap), ShouldKeep)); } for (const auto &CondCap : CondCapabilities) { const SPIRVId Condition = CondCap.first.first; const Capability Cap = CondCap.first.second; const bool ShouldKeep = CondCap.second; if (ShouldKeep) { BM->addCapability(Cap); } else { // In case the capability was auto-added by other instruction BM->eraseCapability(Cap); } BM->eraseConditionalCapability(Condition, Cap); } // Specialize conditional extensions std::vector, bool>> CondExtensions; for (const auto &CondExt : BM->getConditionalExtensions()) { const SPIRVId Cond = CondExt.first; const std::string Ext = CondExt.second; bool ShouldKeep = false; if (!evaluateConstant(BM, Cond, ShouldKeep, ErrMsg)) { return false; } CondExtensions.emplace_back( std::make_pair(std::make_pair(Cond, Ext), ShouldKeep)); } for (const auto &CondExt : CondExtensions) { const auto Ext = CondExt.first; const bool ShouldKeep = CondExt.second; if (ShouldKeep) { BM->getExtension().insert(Ext.second); } else { // In case the extension was auto-added by other instruction BM->getExtension().erase(Ext.second); } BM->getConditionalExtensions().erase(Ext); } // Specialize conditional entry points std::vector> CondEPs; for (const auto &CondEP : BM->getConditionalEntryPoints()) { const SPIRVId Cond = CondEP->getCondition(); bool ShouldKeep = false; if (!evaluateConstant(BM, Cond, ShouldKeep, ErrMsg)) { return false; } CondEPs.emplace_back(std::make_pair(Cond, ShouldKeep)); } for (const auto &CondEP : CondEPs) { const SPIRVId Cond = CondEP.first; const bool ShouldKeep = CondEP.second; BM->specializeConditionalEntryPoints(Cond, ShouldKeep); } // Specialize conditional copy object std::vector> ToReplace; for (unsigned IF = 0; IF < BM->getNumFunctions(); ++IF) { const auto *Fun = BM->getFunction(IF); for (unsigned IB = 0; IB < Fun->getNumBasicBlock(); ++IB) { const auto *BB = Fun->getBasicBlock(IB); for (unsigned II = 0; II < BB->getNumInst(); ++II) { auto *Inst = BB->getInst(II); if (Inst->getOpCode() == OpConditionalCopyObjectINTEL) { const auto OperandIds = static_cast(Inst) ->getOperandIds(); llvm::Optional ITrue; for (unsigned IO = 0; IO < OperandIds.size(); IO += 2) { const auto CondId = OperandIds[IO]; bool Res; if (!evaluateConstant(BM, CondId, Res, ErrMsg)) { return false; } if (Res) { // Stop at the first condition operand that evaluates to true ITrue = IO; break; } } if (!ITrue.has_value()) { ErrMsg = "At least one conditional of OpConditionalCopyObjectINTEL " "must be true. This could mean that all function variants " "have been removed."; return false; } ToReplace.emplace_back( std::make_pair(Inst, OperandIds[ITrue.value() + 1])); } } } } for (auto &It : ToReplace) { auto *OldInst = It.first; auto *BB = OldInst->getBasicBlock(); auto *NextInst = OldInst->getNext(); auto *Operand = BM->getValue(It.second); auto *NewInst = new SPIRVCopyObject(OldInst->getType(), OldInst->getId(), Operand, BB); BM->eraseInstruction(OldInst, BB); BB->addInstruction(NewInst, NextInst); } // Specialize IDs annotated with ConditionalINTEL decorations auto *Decors = BM->getDecorateVec(); std::vector IdsToRemove; for (const auto &D : *Decors) { if (D->getDecorateKind() == DecorationConditionalINTEL) { const SPIRVId ConstId = static_cast(D->getLiteral(0)); bool ShouldKeep = false; if (!evaluateConstant(BM, ConstId, ShouldKeep, ErrMsg)) { return false; } if (!ShouldKeep) { IdsToRemove.push_back(D->getTargetId()); } } } for (const auto &Id : IdsToRemove) { if (!BM->eraseReferencesOfInst(Id)) { ErrMsg = "Error removing references of instruction decorated with " "ConditionalINTEL"; return false; } auto *Val = BM->getValue(Id); if (Val->getOpCode() == OpFunctionCall) { auto *Call = static_cast(Val); auto *BB = Call->getBasicBlock(); BM->eraseInstruction(Call, BB); } else if (Val->getOpCode() == OpFunction) { const auto *Fun = static_cast(Val); for (unsigned I = 0; I < Fun->getNumArguments(); ++I) { const auto ArgId = Fun->getArgumentId(I); if (!BM->eraseReferencesOfInst(ArgId)) { ErrMsg = "Error erasing references of argument of a function " "annotated with ConditionalINTEL"; return false; } } for (const auto &VarId : Fun->getVariables()) { if (!BM->eraseReferencesOfInst(VarId)) { ErrMsg = "Error erasing references of variable within function " "annotated with ConditionalINTEL"; return false; } } for (unsigned IB = 0; IB < Fun->getNumBasicBlock(); ++IB) { const auto *const BB = Fun->getBasicBlock(IB); for (unsigned II = 0; II < BB->getNumInst(); ++II) { const auto *const Inst = BB->getInst(II); if (Inst->hasId()) { const auto InstId = Inst->getId(); if (!BM->eraseReferencesOfInst(InstId)) { ErrMsg = "Error erasing references of instruction within " "function annotated with ConditionalINTEL"; return false; } } } if (!BM->eraseReferencesOfInst(BB->getId())) { ErrMsg = "Error erasing references of basic block label within " "function annotated with ConditionalINTEL"; return false; } } erase_if(*BM->getFuncVec(), [Id](auto F) { return F->getId() == Id; }); } else if (Val->getOpCode() == OpVariable || isTypeOpCode(Val->getOpCode()) || Val->getOpCode() == OpExtInstImport || isConstantOpCode(Val->getOpCode()) || Val->getOpCode() == OpAsmINTEL || Val->getOpCode() == OpAsmTargetINTEL) { if (!BM->eraseValue(Val)) { ErrMsg = "Error erasing value annotated with ConditionalINTEL"; return false; } } else { ErrMsg = "Unsupported instruction annotated with ConditionalINTEL"; return false; } } // Remove any leftover ConditionalINTEL decorations erase_if(*Decors, [](auto D) { return D->getDecorateKind() == DecorationConditionalINTEL; }); // Remove capabilities/extensions of SPV_INTEL_function_variants BM->eraseCapability(CapabilityFunctionVariantsINTEL); BM->eraseCapability(CapabilitySpecConditionalINTEL); BM->getExtension().erase("SPV_INTEL_function_variants"); return true; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVFnVar.h000066400000000000000000000350111506646577400225670ustar00rootroot00000000000000//===- SPIRVFnVar.cpp -===// // // The LLVM/SPIRV Translator // // Copyright (c) 2025 The Khronos Group Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines entries from the SPV_INTEL_function_variants extension. It /// also provides a function for specializing a multi-target module into a /// targeted one. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVFNVAR_H #define SPIRV_LIBSPIRV_SPIRVFNVAR_H #include "SPIRVEntry.h" #include "SPIRVInstruction.h" #include "SPIRVValue.h" #if _SPIRVDBG #include #endif namespace SPIRV { // Specialize multitarget module into a targeted one according to // SPV_INTEL_function_variants. // // This is a SPIR-V to SPIR-V transform. After the transform, the module should // not contain any notion of SPV_INTEL_function_variants and can be processed // even by consumers that do not support this extension. bool specializeFnVariants(SPIRVModule *BM, std::string &ErrMsg); // Below are entries defined by the extension: class SPIRVConditionalEntryPointINTEL : public SPIRVAnnotation { public: static const SPIRVWord FixedWC = 5; SPIRVConditionalEntryPointINTEL(SPIRVModule *TheModule, SPIRVId Condition, SPIRVExecutionModelKind TheExecModel, SPIRVId TheId, const std::string &TheName, std::vector Variables) : SPIRVAnnotation(OpConditionalEntryPointINTEL, TheModule->get(TheId), getSizeInWords(TheName) + Variables.size() + 4), Condition(Condition), ExecModel(TheExecModel), Name(TheName), Variables(Variables) {} SPIRVConditionalEntryPointINTEL() : SPIRVAnnotation(OpConditionalEntryPointINTEL) {} SPIRVId getCondition() const { return Condition; } SPIRVExecutionModelKind getExecModel() const { return ExecModel; } std::string getName() const { return Name; } std::vector getVariables() const { return Variables; } protected: void encode(spv_ostream &O) const override { getEncoder(O) << Condition << ExecModel << Target << Name << Variables; } void decode(std::istream &I) override { getDecoder(I) >> Condition >> ExecModel >> Target >> Name; Variables.resize(WordCount - FixedWC - getSizeInWords(Name) + 1); getDecoder(I) >> Variables; Module->setName(getOrCreateTarget(), Name); Module->addConditionalEntryPoint(Condition, ExecModel, Target, Name, Variables); } private: SPIRVId Condition; SPIRVExecutionModelKind ExecModel = ExecutionModelMax; std::string Name; std::vector Variables; }; class SPIRVConditionalExtensionINTEL : public SPIRVEntryNoId { public: SPIRVConditionalExtensionINTEL(SPIRVModule *M, SPIRVId C, const std::string &SS) : SPIRVEntryNoId(M, 2 + getSizeInWords(SS)), Condition(C), S(SS) {} SPIRVConditionalExtensionINTEL() {} std::string getExtensionName() const { return S; } SPIRVId getCondition() const { return Condition; } protected: void encode(spv_ostream &O) const override { getEncoder(O) << Condition << S; } void decode(std::istream &I) override { getDecoder(I) >> Condition >> S; Module->getConditionalExtensions().insert(std::make_pair(Condition, S)); } private: SPIRVId Condition; std::string S; }; class SPIRVConditionalCapabilityINTEL : public SPIRVEntryNoId { public: static const SPIRVWord FixedWC = 3; SPIRVConditionalCapabilityINTEL(SPIRVModule *M, SPIRVId C, SPIRVCapabilityKind K) : SPIRVEntryNoId(M, 3), Condition(C), Kind(K) { updateModuleVersion(); } SPIRVConditionalCapabilityINTEL() {} SPIRVId getCondition() const { return Condition; } protected: void encode(spv_ostream &O) const override { getEncoder(O) << Condition << Kind; } void decode(std::istream &I) override { getDecoder(I) >> Condition >> Kind; Module->addConditionalCapability(Condition, Kind); } private: SPIRVId Condition; SPIRVCapabilityKind Kind; }; class SPIRVConditionalCopyObjectINTEL : public SPIRVInstruction { public: const static Op OC = OpConditionalCopyObjectINTEL; const static SPIRVWord FixedWordCount = 3; // Complete constructor SPIRVConditionalCopyObjectINTEL(SPIRVType *TheType, SPIRVId TheId, const std::vector &TheConstituents, SPIRVBasicBlock *TheBB) : SPIRVInstruction(FixedWordCount + TheConstituents.size(), OC, TheType, TheId, TheBB), Constituents(TheConstituents) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVConditionalCopyObjectINTEL() : SPIRVInstruction(OC) {} std::vector getOperandIds() { return Constituents; } std::vector getOperands() override { return getValues(Constituents); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Constituents.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC3(Type, Id, Constituents) void validate() const override { SPIRVInstruction::validate(); const size_t TypeOpCode = this->getType()->getOpCode(); assert(TypeOpCode != OpTypeVoid && "Conditional copy type cannot be void"); (void)(TypeOpCode); assert(Constituents.size() % 2 == 0 && "Conditional copy requires condition-operand pairs"); assert(Constituents.size() >= 2 && "Conditional copy requires at least one condition-operand pair"); } private: std::vector Constituents; }; class SPIRVSpecConstantTargetINTEL : public SPIRVValue { public: constexpr static SPIRVWord FixedWC = 4; constexpr static spv::Op OC = OpSpecConstantTargetINTEL; // Complete constructor SPIRVSpecConstantTargetINTEL(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, SPIRVWord TheTarget, const std::vector TheFeatures) : SPIRVValue(M, TheFeatures.size() + FixedWC, OC, TheType, TheId) { Features = TheFeatures; Target = TheTarget; NumWords = TheFeatures.size() + FixedWC; validate(); } // Incomplete constructor SPIRVSpecConstantTargetINTEL() : SPIRVValue(OC) {} SPIRVWord getTarget() const { return Target; } bool matchesDevice() { llvm::Optional DeviceTarget = getModule()->getFnVarTarget(); const std::vector DeviceFeatures = getModule()->getFnVarFeatures(); bool Res = true; if (DeviceTarget.has_value() && DeviceTarget.value() != Target) { Res = false; } if (!DeviceFeatures.empty()) { for (const auto &Feature : Features) { if (std::find(DeviceFeatures.cbegin(), DeviceFeatures.cend(), Feature) == DeviceFeatures.cend()) { Res = false; } } } SPIRVDBG( spvdbgs() << "[FnVar] match instr Target: "; spvdbgs() << std::setw(4) << Target; spvdbgs() << ", Features:"; if (Features.empty()) { spvdbgs() << " none"; } else { for (const auto &Feat : Features) { spvdbgs() << " " << Feat; } } spvdbgs() << " | ID: %" << getId() << std::endl; spvdbgs() << "[FnVar] device Target: "; if (!DeviceTarget.has_value()) { spvdbgs() << "none"; } else { spvdbgs() << std::setw(4) << DeviceTarget.value(); } spvdbgs() << ", Features:"; for (const auto &Feat : DeviceFeatures) { spvdbgs() << " " << Feat; } spvdbgs() << std::endl; spvdbgs() << "[FnVar] result: " << Res << std::endl;); return Res; } protected: _SPIRV_DEF_ENCDEC4(Type, Id, Target, Features); void setWordCount(SPIRVWord WordCount) override { SPIRVEntry::setWordCount(WordCount); Features.resize(WordCount - FixedWC); NumWords = WordCount - FixedWC; } private: unsigned NumWords; std::vector Features; SPIRVWord Target; }; class SPIRVSpecConstantArchitectureINTEL : public SPIRVValue { public: constexpr static SPIRVWord FixedWC = 7; constexpr static spv::Op OC = OpSpecConstantArchitectureINTEL; // Complete constructor SPIRVSpecConstantArchitectureINTEL(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, SPIRVWord TheCategory, SPIRVWord TheFamily, spv::Op TheCmpOp, SPIRVWord TheArchitecture) : SPIRVValue(M, FixedWC, OC, TheType, TheId) { Category = TheCategory; Family = TheFamily; CmpOp = TheCmpOp; Architecture = TheArchitecture; validate(); } // Incomplete constructor SPIRVSpecConstantArchitectureINTEL() : SPIRVValue(OC) {} SPIRVWord getCategory() { return Category; } SPIRVWord getFamily() { return Family; } spv::Op getCmpOp() { return CmpOp; } SPIRVWord getArchitecture() { return Architecture; } bool matchesDevice() { llvm::Optional DeviceCategory = getModule()->getFnVarCategory(); llvm::Optional DeviceFamily = getModule()->getFnVarFamily(); const llvm::Optional DeviceArchitecture = getModule()->getFnVarArch(); bool Res = true; if (DeviceCategory.has_value() && DeviceCategory.value() != Category) { Res = false; } if (DeviceFamily.has_value() && DeviceFamily.value() != Family) { Res = false; } if (DeviceArchitecture.has_value()) { switch (CmpOp) { case OpIEqual: Res = DeviceArchitecture == Architecture; break; case OpINotEqual: Res = DeviceArchitecture != Architecture; break; case OpULessThan: Res = DeviceArchitecture < Architecture; break; case OpULessThanEqual: Res = DeviceArchitecture <= Architecture; break; case OpUGreaterThan: Res = DeviceArchitecture > Architecture; break; case OpUGreaterThanEqual: Res = DeviceArchitecture >= Architecture; break; default: assert(false && "Invalid checked CmpOp"); Res = false; break; } } SPIRVDBG( spvdbgs() << "[FnVar] match instr Category: " << std::setw(4) << Category << ", Family: " << std::setw(4) << Family << ", CmpOp: " << std::setw(4) << CmpOp << ", Architecture: " << std::setw(4) << Architecture << " | ID: %" << getId() << std::endl; spvdbgs() << "[FnVar] device Category: "; if (!DeviceCategory.has_value()) { spvdbgs() << "none"; } else { spvdbgs() << std::setw(4) << DeviceCategory.value(); } spvdbgs() << ", Family: "; if (!DeviceFamily.has_value()) { spvdbgs() << "none"; } else { spvdbgs() << std::setw(4) << DeviceFamily.value(); } spvdbgs() << ", Architecture: "; if (!DeviceArchitecture.has_value()) { spvdbgs() << "none"; } else { spvdbgs() << std::setw(4) << DeviceArchitecture.value(); } spvdbgs() << std::endl; spvdbgs() << "[FnVar] result: " << Res << std::endl;); return Res; } protected: _SPIRV_DEF_ENCDEC6(Type, Id, Category, Family, CmpOp, Architecture); private: SPIRVWord Category; SPIRVWord Family; spv::Op CmpOp; SPIRVWord Architecture; }; class SPIRVSpecConstantCapabilitiesINTEL : public SPIRVValue { public: constexpr static SPIRVWord FixedWC = 3; constexpr static spv::Op OC = OpSpecConstantCapabilitiesINTEL; // Complete constructor SPIRVSpecConstantCapabilitiesINTEL( SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, const std::vector TheCapabilities) : SPIRVValue(M, TheCapabilities.size() + FixedWC, OC, TheType, TheId) { Capabilities = TheCapabilities; NumWords = TheCapabilities.size() + FixedWC; validate(); } // Incomplete constructor SPIRVSpecConstantCapabilitiesINTEL() : SPIRVValue(OC) {} std::vector getCapabilities() const { return Capabilities; } bool matchesDevice() { const std::vector DeviceCapabilities = getModule()->getFnVarCapabilities(); bool Res = true; if (!DeviceCapabilities.empty()) { for (const auto &Capability : Capabilities) { if (std::find(DeviceCapabilities.cbegin(), DeviceCapabilities.cend(), Capability) == DeviceCapabilities.cend()) { Res = false; } } } SPIRVDBG( spvdbgs() << "[FnVar] match instr Capabilities: "; if (Capabilities.empty()) { spvdbgs() << "none"; } else { for (const auto &Cap : Capabilities) { spvdbgs() << " " << Cap; } } spvdbgs() << " | ID: %" << getId() << std::endl; spvdbgs() << "[FnVar] device Capabilities: "; for (const auto &Cap : DeviceCapabilities) { spvdbgs() << " " << Cap; } spvdbgs() << std::endl; spvdbgs() << "[FnVar] result: " << Res << std::endl;); return Res; } protected: _SPIRV_DEF_ENCDEC3(Type, Id, Capabilities); void validate() const override { SPIRVValue::validate(); if (Capabilities.size() < 1) { std::stringstream SS; SS << "Id: " << Id << ", OpCode: " << OpCodeNameMap::map(OpCode) << ", Name: \"" << Name << "\". Expected at least one capability.\n"; getErrorLog().checkError(false, SPIRVEC_InvalidNumberOfOperands, SS.str()); } } void setWordCount(SPIRVWord WordCount) override { SPIRVEntry::setWordCount(WordCount); Capabilities.resize(WordCount - FixedWC); NumWords = WordCount - FixedWC; } private: unsigned NumWords; std::vector Capabilities; }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVFNVAR_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVFunction.cpp000066400000000000000000000171011506646577400236730ustar00rootroot00000000000000//===- SPIRVFunction.cpp - Class to represent a SPIR-V Function --- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file implements Function class for SPIRV. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #include "SPIRVFunction.h" #include "SPIRVBasicBlock.h" #include "SPIRVEntry.h" #include "SPIRVInstruction.h" #include "SPIRVStream.h" #include #include using namespace SPIRV; SPIRVFunctionParameter::SPIRVFunctionParameter(SPIRVType *TheType, SPIRVId TheId, SPIRVFunction *TheParent, unsigned TheArgNo) : SPIRVValue(TheParent->getModule(), 3, OpFunctionParameter, TheType, TheId), ParentFunc(TheParent), ArgNo(TheArgNo) { validate(); } void SPIRVFunctionParameter::foreachAttr( std::function Func) { auto Locs = Decorates.equal_range(DecorationFuncParamAttr); for (auto I = Locs.first, E = Locs.second; I != E; ++I) { auto Attr = static_cast(I->second->getLiteral(0)); assert(isValid(Attr)); Func(Attr); } } SPIRVDecoder SPIRVFunction::getDecoder(std::istream &IS) { return SPIRVDecoder(IS, *this); } void SPIRVFunction::encode(spv_ostream &O) const { getEncoder(O) << Type << Id << FCtrlMask << FuncType; } void SPIRVFunction::encodeChildren(spv_ostream &O) const { O << SPIRVNL(); for (auto &I : Parameters) O << *I; O << SPIRVNL(); for (auto &I : BBVec) O << *I; O << SPIRVFunctionEnd(); } void SPIRVFunction::encodeExecutionModes(spv_ostream &O) const { for (auto &I : ExecModes) O << *I.second; } void SPIRVFunction::decode(std::istream &I) { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Type >> Id >> FCtrlMask >> FuncType; Module->addFunction(this); SPIRVDBG(spvdbgs() << "Decode function: " << Id << '\n'); Decoder.getWordCountAndOpCode(); while (!I.eof()) { if (Decoder.OpCode == OpFunctionEnd) break; switch (Decoder.OpCode) { case OpFunctionParameter: { auto Param = static_cast(Decoder.getEntry()); assert(Param); Module->add(Param); Param->setParent(this); Parameters.push_back(Param); Decoder.getWordCountAndOpCode(); break; } case OpLabel: { if (!decodeBB(Decoder)) return; break; } default: assert(0 && "Invalid SPIRV format"); } } } /// Decode basic block and contained instructions. /// Do it here instead of in BB:decode to avoid back track in input stream. bool SPIRVFunction::decodeBB(SPIRVDecoder &Decoder) { SPIRVBasicBlock *BB = static_cast(Decoder.getEntry()); assert(BB); addBasicBlock(BB); SPIRVDBG(spvdbgs() << "Decode BB: " << BB->getId() << '\n'); Decoder.setScope(BB); SPIRVEntry *DebugScope = nullptr; while (Decoder.getWordCountAndOpCode()) { if (Decoder.OpCode == OpFunctionEnd || Decoder.OpCode == OpLabel) { break; } if (Decoder.OpCode == OpNoLine || Decoder.OpCode == OpNop) { continue; } SPIRVEntry *Entry = Decoder.getEntry(); if (Decoder.OpCode == OpLine) { std::shared_ptr L(static_cast(Entry)); Module->setCurrentLine(L); continue; } if (!Module->getErrorLog().checkError(Entry->isImplemented(), SPIRVEC_UnimplementedOpCode, std::to_string(Entry->getOpCode()))) { // Bail out if the opcode is not implemented. Module->setInvalid(); return false; } auto *Inst = static_cast(Entry); assert(Inst); if (Inst->getOpCode() == OpUndef) { Module->add(Inst); } else if (Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::DebugNoLine) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugNoLine)) { continue; } else if (Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::DebugLine) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugLine)) { std::shared_ptr DL(static_cast(Inst)); Module->setCurrentDebugLine(DL); } else { if (Inst->isExtInst(SPIRVEIS_Debug, SPIRVDebug::Scope) || Inst->isExtInst(SPIRVEIS_OpenCL_DebugInfo_100, SPIRVDebug::Scope) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::Scope) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::Scope)) { DebugScope = Inst; } else if (Inst->isExtInst(SPIRVEIS_Debug, SPIRVDebug::NoScope) || Inst->isExtInst(SPIRVEIS_OpenCL_DebugInfo_100, SPIRVDebug::NoScope) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::NoScope) || Inst->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::NoScope)) { DebugScope = nullptr; } else { Inst->setDebugScope(DebugScope); } BB->addInstruction(Inst); } } Decoder.setScope(this); return true; } void SPIRVFunction::foreachReturnValueAttr( std::function Func) { auto Locs = Decorates.equal_range(DecorationFuncParamAttr); for (auto I = Locs.first, E = Locs.second; I != E; ++I) { auto Attr = static_cast(I->second->getLiteral(0)); assert(isValid(Attr)); Func(Attr); } } SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVFunction.h000066400000000000000000000170511506646577400233440ustar00rootroot00000000000000//===- SPIRVFunction.h - Class to represent a SPIR-V function ---*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file defines Function class for SPIRV. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVFUNCTION_H #define SPIRV_LIBSPIRV_SPIRVFUNCTION_H #include "SPIRVBasicBlock.h" #include "SPIRVValue.h" #include namespace SPIRV { class BIFunction; class SPIRVDecoder; class SPIRVFunctionParameter : public SPIRVValue { public: SPIRVFunctionParameter(SPIRVType *TheType, SPIRVId TheId, SPIRVFunction *TheParent, unsigned TheArgNo); SPIRVFunctionParameter() : SPIRVValue(OpFunctionParameter), ParentFunc(nullptr), ArgNo(0) {} unsigned getArgNo() const { return ArgNo; } void foreachAttr(std::function); void addAttr(SPIRVFuncParamAttrKind Kind) { addDecorate(new SPIRVDecorate(DecorationFuncParamAttr, this, Kind)); } void setParent(SPIRVFunction *Parent) { ParentFunc = Parent; } bool hasAttr(SPIRVFuncParamAttrKind Kind) const { return getDecorate(DecorationFuncParamAttr).count(Kind); } bool isByVal() const { return hasAttr(FunctionParameterAttributeByVal); } bool isZext() const { return hasAttr(FunctionParameterAttributeZext); } SPIRVCapVec getRequiredCapability() const override { if (hasLinkageType() && getLinkageType() == LinkageTypeImport) return getVec(CapabilityLinkage); return SPIRVCapVec(); } protected: void validate() const override { SPIRVValue::validate(); assert(ParentFunc && "Invalid parent function"); } _SPIRV_DEF_ENCDEC2(Type, Id) private: SPIRVFunction *ParentFunc; unsigned ArgNo; }; class SPIRVFunction : public SPIRVValue, public SPIRVComponentExecutionModes { public: // Complete constructor. It does not construct basic blocks. SPIRVFunction(SPIRVModule *M, SPIRVTypeFunction *FunctionType, SPIRVId TheId) : SPIRVValue(M, 5, OpFunction, FunctionType->getReturnType(), TheId), FuncType(FunctionType), FCtrlMask(FunctionControlMaskNone) { addAllArguments(TheId + 1); validate(); } // Incomplete constructor SPIRVFunction() : SPIRVValue(OpFunction), FuncType(NULL), FCtrlMask(FunctionControlMaskNone) {} SPIRVDecoder getDecoder(std::istream &IS) override; SPIRVTypeFunction *getFunctionType() const { return FuncType; } SPIRVWord getFuncCtlMask() const { return FCtrlMask; } size_t getNumBasicBlock() const { return BBVec.size(); } SPIRVBasicBlock *getBasicBlock(size_t I) const { return BBVec[I]; } size_t getNumArguments() const { return getFunctionType()->getNumParameters(); } SPIRVId getArgumentId(size_t I) const { return Parameters[I]->getId(); } const std::vector getVariables() const { std::vector Ids; for (auto Variable : Variables) Ids.push_back(Variable->getId()); return Ids; } void addVariable(const SPIRVValue *Variable) { Variables.push_back(Variable); } SPIRVFunctionParameter *getArgument(size_t I) const { return Parameters[I]; } void foreachArgument(std::function Func) { for (size_t I = 0, E = getNumArguments(); I != E; ++I) Func(getArgument(I)); } void foreachReturnValueAttr(std::function); void setFunctionControlMask(SPIRVWord Mask) { FCtrlMask = Mask; } void takeExecutionModes(SPIRVForward *Forward) { ExecModes = std::move(Forward->ExecModes); } // Assume BB contains valid Id. SPIRVBasicBlock *addBasicBlock(SPIRVBasicBlock *BB) { Module->add(BB); BB->setParent(this); BBVec.push_back(BB); return BB; } void encodeChildren(spv_ostream &) const override; void encodeExecutionModes(spv_ostream &) const; _SPIRV_DCL_ENCDEC void validate() const override { SPIRVValue::validate(); validateFunctionControlMask(FCtrlMask); assert(FuncType && "Invalid func type"); } private: SPIRVFunctionParameter *addArgument(unsigned TheArgNo, SPIRVId TheId) { SPIRVFunctionParameter *Arg = new SPIRVFunctionParameter( getFunctionType()->getParameterType(TheArgNo), TheId, this, TheArgNo); Module->add(Arg); Parameters.push_back(Arg); return Arg; } void addAllArguments(SPIRVId FirstArgId) { for (size_t I = 0, E = getFunctionType()->getNumParameters(); I != E; ++I) addArgument(I, FirstArgId + I); } bool decodeBB(SPIRVDecoder &); SPIRVTypeFunction *FuncType; // Function type SPIRVWord FCtrlMask; // Function control mask std::vector Parameters; std::vector Variables; typedef std::vector SPIRVLBasicBlockVector; SPIRVLBasicBlockVector BBVec; }; typedef SPIRVEntryOpCodeOnly SPIRVFunctionEnd; class SPIRVConstantFunctionPointerINTEL : public SPIRVValue { const static Op OC = OpConstantFunctionPointerINTEL; const static SPIRVWord FixedWordCount = 4; public: SPIRVConstantFunctionPointerINTEL(SPIRVId TheId, SPIRVType *TheType, SPIRVFunction *TheFunction, SPIRVModule *M) : SPIRVValue(M, FixedWordCount, OC, TheType, TheId), TheFunction(TheFunction->getId()) { validate(); } SPIRVConstantFunctionPointerINTEL() : SPIRVValue(OC), TheFunction(SPIRVID_INVALID) {} SPIRVFunction *getFunction() const { return get(TheFunction); } _SPIRV_DEF_ENCDEC3(Type, Id, TheFunction) void validate() const override { SPIRVValue::validate(); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_function_pointers; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityFunctionPointersINTEL); } protected: SPIRVId TheFunction; }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVFUNCTION_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVInstruction.cpp000066400000000000000000000253001506646577400244270ustar00rootroot00000000000000//===- SPIRVInstruction.cpp -Class to represent SPIR-V instruction - C++ --===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements SPIR-V instructions. /// //===----------------------------------------------------------------------===// #include "SPIRVInstruction.h" #include "SPIRVBasicBlock.h" #include "SPIRVFunction.h" #include "SPIRVInternal.h" #include namespace SPIRV { // Complete constructor for instruction with type and id SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB) : SPIRVValue(TheBB->getModule(), TheWordCount, TheOC, TheType, TheId), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB, SPIRVModule *TheBM) : SPIRVValue(TheBM, TheWordCount, TheOC, TheType, TheId), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } // Complete constructor for instruction with id but no type SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVId TheId, SPIRVBasicBlock *TheBB) : SPIRVValue(TheBB->getModule(), TheWordCount, TheOC, TheId), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } // Complete constructor for instruction without type and id SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVBasicBlock *TheBB) : SPIRVValue(TheBB->getModule(), TheWordCount, TheOC), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } // Complete constructor for instruction with type but no id SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVBasicBlock *TheBB) : SPIRVValue(TheBB->getModule(), TheWordCount, TheOC, TheType), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } // Special constructor for debug instruction SPIRVInstruction::SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVModule *TheBM, SPIRVBasicBlock *TheBB) : SPIRVValue(TheBM, TheWordCount, TheOC, TheType, TheId), BB(TheBB), DebugScope(nullptr) { SPIRVInstruction::validate(); } void SPIRVInstruction::setParent(SPIRVBasicBlock *TheBB) { assert(TheBB && "Invalid BB"); if (BB == TheBB) return; assert(BB == NULL && "BB cannot change parent"); BB = TheBB; } void SPIRVInstruction::setScope(SPIRVEntry *Scope) { assert(Scope && Scope->getOpCode() == OpLabel && "Invalid scope"); setParent(static_cast(Scope)); } SPIRVFunctionCall::SPIRVFunctionCall(SPIRVId TheId, SPIRVFunction *TheFunction, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVFunctionCallGeneric(TheFunction->getFunctionType()->getReturnType(), TheId, TheArgs, BB), FunctionId(TheFunction->getId()) { validate(); } void SPIRVFunctionCall::validate() const { SPIRVFunctionCallGeneric::validate(); } SPIRVFunctionPointerCallINTEL::SPIRVFunctionPointerCallINTEL( SPIRVId TheId, SPIRVValue *TheCalledValue, SPIRVType *TheReturnType, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVFunctionCallGeneric(TheReturnType, TheId, TheArgs, BB), CalledValueId(TheCalledValue->getId()) { validate(); } void SPIRVFunctionPointerCallINTEL::validate() const { SPIRVFunctionCallGeneric::validate(); } // ToDo: Each instruction should implement this function std::vector SPIRVInstruction::getOperands() { std::vector Empty; assert(0 && "not supported"); return Empty; } std::vector SPIRVInstruction::getOperandTypes(const std::vector &Ops) { std::vector Tys; for (auto &I : Ops) { SPIRVType *Ty = nullptr; if (I->getOpCode() == OpFunction) Ty = reinterpret_cast(I)->getFunctionType(); else if (I->getOpCode() == OpTypeCooperativeMatrixKHR) Ty = reinterpret_cast(I); else Ty = I->getType(); Tys.push_back(Ty); } return Tys; } std::vector SPIRVInstruction::getOperandTypes() { return getOperandTypes(getOperands()); } void SPIRVImageInstBase::setOpWords(const std::vector &OpsArg) { std::vector Ops = OpsArg; // If the Image Operands field has the SignExtend or ZeroExtend bit set, // either raise the minimum SPIR-V version to 1.4, or drop the operand // if SPIR-V 1.4 cannot be emitted. const size_t ImgOpsIndex = getImageOperandsIndex(OpCode); if (ImgOpsIndex != ~0U && ImgOpsIndex < Ops.size()) { SPIRVWord ImgOps = Ops[ImgOpsIndex]; unsigned SignZeroExtMasks = ImageOperandsMask::ImageOperandsSignExtendMask | ImageOperandsMask::ImageOperandsZeroExtendMask; if (ImgOps & SignZeroExtMasks) { SPIRVModule *M = getModule(); if (M->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) { M->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); } else { // Drop SignExtend/ZeroExtend if we cannot use SPIR-V 1.4. Ops[ImgOpsIndex] &= ~SignZeroExtMasks; if (Ops[ImgOpsIndex] == 0) { // Drop the Image Operands if SignExtend/ZeroExtend was the only // bit set. Ops.pop_back(); } } } } SPIRVInstTemplateBase::setOpWords(Ops); } bool isSpecConstantOpAllowedOp(Op OC) { static SPIRVWord Table[] = { OpSConvert, OpFConvert, OpConvertFToS, OpConvertSToF, OpConvertFToU, OpConvertUToF, OpUConvert, OpConvertPtrToU, OpConvertUToPtr, OpGenericCastToPtr, OpPtrCastToGeneric, OpCrossWorkgroupCastToPtrINTEL, OpPtrCastToCrossWorkgroupINTEL, OpBitcast, OpQuantizeToF16, OpSNegate, OpNot, OpIAdd, OpISub, OpIMul, OpUDiv, OpSDiv, OpUMod, OpSRem, OpSMod, OpShiftRightLogical, OpShiftRightArithmetic, OpShiftLeftLogical, OpBitwiseOr, OpBitwiseXor, OpBitwiseAnd, OpFNegate, OpFAdd, OpFSub, OpFMul, OpFDiv, OpFRem, OpFMod, OpVectorShuffle, OpCompositeExtract, OpCompositeInsert, OpLogicalOr, OpLogicalAnd, OpLogicalNot, OpLogicalEqual, OpLogicalNotEqual, OpSelect, OpIEqual, OpINotEqual, OpULessThan, OpSLessThan, OpUGreaterThan, OpSGreaterThan, OpULessThanEqual, OpSLessThanEqual, OpUGreaterThanEqual, OpSGreaterThanEqual, OpAccessChain, OpInBoundsAccessChain, OpPtrAccessChain, OpInBoundsPtrAccessChain, }; static std::unordered_set Allow(std::begin(Table), std::end(Table)); return Allow.count(OC); } SPIRVSpecConstantOp *createSpecConstantOpInst(SPIRVInstruction *Inst) { auto OC = Inst->getOpCode(); assert(isSpecConstantOpAllowedOp(OC) && "Op code not allowed for OpSpecConstantOp"); std::vector Ops; // CompositeExtract/Insert operations use zero-based numbering for their // indexes (containted in instruction operands). All their operands are // Literals, so we can pass them as is for further handling. if (OC == OpCompositeExtract || OC == OpCompositeInsert) { auto *SPIRVInst = static_cast(Inst); Ops = SPIRVInst->getOpWords(); } else { Ops = Inst->getIds(Inst->getOperands()); } Ops.insert(Ops.begin(), OC); return static_cast(SPIRVSpecConstantOp::create( OpSpecConstantOp, Inst->getType(), Inst->getId(), Ops, nullptr, Inst->getModule())); } SPIRVInstruction *createInstFromSpecConstantOp(SPIRVSpecConstantOp *Inst) { assert(Inst->getOpCode() == OpSpecConstantOp && "Not OpSpecConstantOp"); auto Ops = Inst->getOpWords(); auto OC = static_cast(Ops[0]); assert(isSpecConstantOpAllowedOp(OC) && "Op code not allowed for OpSpecConstantOp"); auto *Const = Inst->getOperand(1); // LLVM would eliminate a bitcast from a function pointer in a constexpr // context. Cut this short here to avoid necessity to align address spaces if (OC == OpBitcast && Const->getOpCode() == OpConstantFunctionPointerINTEL) return static_cast(Const); Ops.erase(Ops.begin(), Ops.begin() + 1); auto *BM = Inst->getModule(); auto *RetInst = SPIRVInstTemplateBase::create( OC, Inst->getType(), Inst->getId(), Ops, nullptr, BM); // Instruction that creates from OpSpecConstantOp has the same Id BM->insertEntryNoId(RetInst); return RetInst; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVInstruction.h000066400000000000000000004357371506646577400241170ustar00rootroot00000000000000//===- SPIRVInstruction.h - Class to represent SPIRV instruction -- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Instruction class for SPIR-V. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H #define SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H #include "SPIRVBasicBlock.h" #include "SPIRVEnum.h" #include "SPIRVFunction.h" #include "SPIRVIsValidEnum.h" #include "SPIRVOpCode.h" #include "SPIRVStream.h" #include "SPIRVValue.h" #include #include #include #include #include #include namespace SPIRV { typedef std::vector ValueVec; typedef std::pair ValueRange; class SPIRVBasicBlock; class SPIRVFunction; bool isSpecConstantOpAllowedOp(Op OC); class SPIRVComponentExecutionScope { public: SPIRVComponentExecutionScope(Scope TheScope = ScopeInvocation) : ExecScope(TheScope) {} Scope ExecScope; }; class SPIRVComponentMemorySemanticsMask { public: SPIRVComponentMemorySemanticsMask(SPIRVWord TheSema = SPIRVWORD_MAX) : MemSema(TheSema) {} SPIRVWord MemSema; }; class SPIRVComponentOperands { public: SPIRVComponentOperands(){}; SPIRVComponentOperands(const std::vector &TheOperands) : Operands(TheOperands){}; SPIRVComponentOperands(std::vector &&TheOperands) : Operands(std::move(TheOperands)){}; std::vector getCompOperands() { return Operands; } std::vector getCompOperandTypes() { std::vector Tys; for (auto &I : getCompOperands()) Tys.push_back(I->getType()); return Tys; } protected: std::vector Operands; }; class SPIRVInstruction : public SPIRVValue { public: // Complete constructor for instruction with type and id SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB); // Complete constructor for instruction with module, type and id SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB, SPIRVModule *TheBM); // Complete constructor for instruction with id but no type SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVId TheId, SPIRVBasicBlock *TheBB); // Complete constructor for instruction without type and id SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVBasicBlock *TheBB); // Complete constructor for instruction with type but no id SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVBasicBlock *TheBB); // Special constructor for debug instruction SPIRVInstruction(unsigned TheWordCount, Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVModule *TheBM, SPIRVBasicBlock *TheBB = nullptr); // Incomplete constructor SPIRVInstruction(Op TheOC = OpNop) : SPIRVValue(TheOC), BB(NULL), DebugScope(nullptr) {} bool isInst() const override { return true; } SPIRVBasicBlock *getParent() const { return BB; } SPIRVInstruction *getPrevious() const { return BB->getPrevious(this); } SPIRVInstruction *getNext() const { return BB->getNext(this); } virtual std::vector getOperands(); std::vector getOperandTypes(); static std::vector getOperandTypes(const std::vector &Ops); void setParent(SPIRVBasicBlock *); void setScope(SPIRVEntry *) override; void addFPRoundingMode(SPIRVFPRoundingModeKind Kind) { addDecorate(DecorationFPRoundingMode, Kind); } void eraseFPRoundingMode() { eraseDecorate(DecorationFPRoundingMode); } void setSaturatedConversion(bool Enable) { if (Enable) addDecorate(DecorationSaturatedConversion); else eraseDecorate(DecorationSaturatedConversion); } bool hasFPRoundingMode(SPIRVFPRoundingModeKind *Kind = nullptr) { SPIRVWord V; auto Found = hasDecorate(DecorationFPRoundingMode, 0, &V); if (Found && Kind) *Kind = static_cast(V); return Found; } bool isSaturatedConversion() { return hasDecorate(DecorationSaturatedConversion) || OpCode == OpSatConvertSToU || OpCode == OpSatConvertUToS; } SPIRVBasicBlock *getBasicBlock() const { return BB; } void setBasicBlock(SPIRVBasicBlock *TheBB) { BB = TheBB; if (TheBB) setModule(TheBB->getModule()); } void setDebugScope(SPIRVEntry *Scope) { DebugScope = Scope; } SPIRVEntry *getDebugScope() const { return DebugScope; } protected: void validate() const override { SPIRVValue::validate(); } private: SPIRVBasicBlock *BB; SPIRVEntry *DebugScope; }; class SPIRVInstTemplateBase : public SPIRVInstruction { public: /// Create an empty instruction. Mainly for getting format information, /// e.g. whether an operand is literal. static SPIRVInstTemplateBase *create(Op TheOC) { auto Inst = static_cast(SPIRVEntry::create(TheOC)); assert(Inst); Inst->init(); return Inst; } /// Create a instruction without operands. static SPIRVInstTemplateBase *create(Op TheOC, SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB, SPIRVModule *TheModule) { auto Inst = create(TheOC); Inst->init(TheType, TheId, TheBB, TheModule); return Inst; } /// Create a complete and valid instruction. static SPIRVInstTemplateBase *create(Op TheOC, SPIRVType *TheType, SPIRVId TheId, const std::vector &TheOps, SPIRVBasicBlock *TheBB, SPIRVModule *TheModule) { auto Inst = create(TheOC); Inst->init(TheType, TheId, TheBB, TheModule); Inst->setOpWords(TheOps); Inst->validate(); return Inst; } SPIRVInstTemplateBase(Op OC = OpNop) : SPIRVInstruction(OC), HasVariWC(false) { init(); } ~SPIRVInstTemplateBase() override {} SPIRVInstTemplateBase *init(SPIRVType *TheType, SPIRVId TheId, SPIRVBasicBlock *TheBB, SPIRVModule *TheModule) { assert((TheBB || TheModule) && "Invalid BB or Module"); if (TheBB) setBasicBlock(TheBB); else { setModule(TheModule); } setId(hasId() ? TheId : SPIRVID_INVALID); setType(hasType() ? TheType : nullptr); return this; } virtual void init() {} virtual void initImpl(Op OC, bool HasId = true, SPIRVWord WC = 0, bool VariWC = false, unsigned Lit1 = ~0U, unsigned Lit2 = ~0U, unsigned Lit3 = ~0U) { OpCode = OC; if (!HasId) { setHasNoId(); setHasNoType(); } if (WC) SPIRVEntry::setWordCount(WC); setHasVariableWordCount(VariWC); addLit(Lit1); addLit(Lit2); addLit(Lit3); } bool isOperandLiteral(unsigned I) const override { return Lit.count(I); } void addLit(unsigned L) { if (L != ~0U) Lit.insert(L); } /// \return Expected number of operands. If the instruction has variable /// number of words, return the minimum. SPIRVWord getExpectedNumOperands() const { assert(WordCount > 0 && "Word count not initialized"); auto Exp = WordCount - 1; if (hasId()) --Exp; if (hasType()) --Exp; return Exp; } virtual void setOpWordsAndValidate(const std::vector &TheOps) { setOpWords(TheOps); validate(); } virtual void setOpWords(const std::vector &TheOps) { SPIRVWord WC = TheOps.size() + 1; if (hasId()) ++WC; if (hasType()) ++WC; if (WordCount) { if (WordCount == WC) { // do nothing } else { assert(HasVariWC && WC >= WordCount && "Invalid word count"); SPIRVEntry::setWordCount(WC); } } else SPIRVEntry::setWordCount(WC); Ops = TheOps; // The required SPIR-V version depends on the operands for some // instructions. updateModuleVersion(); } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); auto NumOps = WordCount - 1; if (hasId()) --NumOps; if (hasType()) --NumOps; Ops.resize(NumOps); } std::vector &getOpWords() { return Ops; } const std::vector &getOpWords() const { return Ops; } SPIRVWord getOpWord(int I) const { return Ops[I]; } /// Get operand as value. /// If the operand is a literal, return it as a uint32 constant. const SPIRVValue *getOpValue(int I) const { return isOperandLiteral(I) ? Module->getLiteralAsConstant(Ops[I]) : getValue(Ops[I]); } SPIRVValue *getOpValue(int I) { return isOperandLiteral(I) ? Module->getLiteralAsConstant(Ops[I]) : getValue(Ops[I]); } std::vector getOperands() override { std::vector VOps; for (size_t I = 0, E = Ops.size(); I != E; ++I) VOps.push_back(getOperand(I)); return VOps; } std::vector getNonLiteralOperands() const override { std::vector Operands; for (size_t I = 0, E = Ops.size(); I < E; ++I) if (!isOperandLiteral(I)) Operands.push_back(getEntry(Ops[I])); return Operands; } virtual const SPIRVValue *getOperand(unsigned I) const { return getOpValue(I); } virtual SPIRVValue *getOperand(unsigned I) { return getOpValue(I); } bool hasExecScope() const { return SPIRV::hasExecScope(OpCode); } bool hasGroupOperation() const { return SPIRV::hasGroupOperation(OpCode); } bool getSPIRVGroupOperation(SPIRVGroupOperationKind &GroupOp) const { if (!hasGroupOperation()) return false; GroupOp = static_cast(Ops[1]); return true; } Scope getExecutionScope() const { if (!hasExecScope()) return ScopeInvocation; return static_cast( static_cast(getValue(Ops[0]))->getZExtIntValue()); } bool hasVariableWordCount() const { return HasVariWC; } void setHasVariableWordCount(bool VariWC) { HasVariWC = VariWC; } protected: void encode(spv_ostream &O) const override { auto E = getEncoder(O); if (hasType()) E << Type; if (hasId()) E << Id; E << Ops; } void decode(std::istream &I) override { auto D = getDecoder(I); if (hasType()) D >> Type; if (hasId()) D >> Id; D >> Ops; } std::vector Ops; bool HasVariWC; std::unordered_set Lit; // Literal operand index }; template class SPIRVInstTemplate : public BT { public: typedef BT BaseTy; SPIRVInstTemplate() { init(); } ~SPIRVInstTemplate() override {} void init() override { this->initImpl(OC, HasId, WC, HasVariableWC, Literal1, Literal2, Literal3); } }; class SPIRVMemoryAccess { public: SPIRVMemoryAccess(const std::vector &TheMemoryAccess) : TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0), AliasScopeInstID(0), NoAliasInstID(0) { memoryAccessUpdate(TheMemoryAccess); } SPIRVMemoryAccess() : TheMemoryAccessMask(0), Alignment(0), SrcAlignment(0), AliasScopeInstID(0), NoAliasInstID(0) {} void memoryAccessUpdate(const std::vector &MemoryAccess) { if (!MemoryAccess.size()) return; assert(MemoryAccess.size() > 0 && "Invalid memory access operand size"); assert(MemoryAccess.size() < 5 && "Invalid memory access operand size"); TheMemoryAccessMask = MemoryAccess[0]; size_t MemAccessNumParam = 1; if (MemoryAccess[0] & MemoryAccessAlignedMask) { assert(MemoryAccess.size() > 1 && "Alignment operand is missing"); Alignment = MemoryAccess[MemAccessNumParam++]; } if (MemoryAccess[0] & MemoryAccessAliasScopeINTELMaskMask) { assert(MemoryAccess.size() > MemAccessNumParam && "Aliasing operand is missing"); AliasScopeInstID = MemoryAccess[MemAccessNumParam++]; } if (MemoryAccess[0] & MemoryAccessNoAliasINTELMaskMask) { assert(MemoryAccess.size() > MemAccessNumParam && "Aliasing operand is missing"); NoAliasInstID = MemoryAccess[MemAccessNumParam++]; } // Exit if there is no second memory operand mask if (MemoryAccess.size() == MemAccessNumParam) return; size_t SecondMaskId = MemAccessNumParam++; if (MemoryAccess[SecondMaskId] & MemoryAccessAlignedMask) { assert(MemoryAccess.size() > MemAccessNumParam && "Alignment operand is missing"); SrcAlignment = MemoryAccess[MemAccessNumParam]; } } SPIRVWord isVolatile() const { return getMemoryAccessMask() & MemoryAccessVolatileMask; } SPIRVWord isNonTemporal() const { return getMemoryAccessMask() & MemoryAccessNontemporalMask; } SPIRVWord isAliasScope() const { return getMemoryAccessMask() & MemoryAccessAliasScopeINTELMaskMask; } SPIRVWord isNoAlias() const { return getMemoryAccessMask() & MemoryAccessNoAliasINTELMaskMask; } SPIRVWord getMemoryAccessMask() const { return TheMemoryAccessMask; } SPIRVWord getAlignment() const { return Alignment; } SPIRVWord getSrcAlignment() const { return SrcAlignment; } SPIRVWord getAliasScopeInstID() const { return AliasScopeInstID; } SPIRVWord getNoAliasInstID() const { return NoAliasInstID; } protected: SPIRVWord TheMemoryAccessMask; SPIRVWord Alignment; SPIRVWord SrcAlignment; SPIRVId AliasScopeInstID; SPIRVId NoAliasInstID; }; class SPIRVVariable : public SPIRVInstruction { public: // Complete constructor for integer constant SPIRVVariable(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheInitializer, const std::string &TheName, SPIRVStorageClassKind TheStorageClass, SPIRVBasicBlock *TheBB, SPIRVModule *TheM) : SPIRVInstruction(TheInitializer ? 5 : 4, OpVariable, TheType, TheId, TheBB, TheM), StorageClass(TheStorageClass) { if (TheInitializer) Initializer.push_back(TheInitializer->getId()); Name = TheName; validate(); } // Incomplete constructor SPIRVVariable() : SPIRVInstruction(OpVariable), StorageClass(StorageClassFunction) {} SPIRVStorageClassKind getStorageClass() const { return StorageClass; } SPIRVValue *getInitializer() const { if (Initializer.empty()) return nullptr; assert(Initializer.size() == 1); return getValue(Initializer[0]); } bool isConstant() const { return hasDecorate(DecorationConstant); } bool isBuiltin(SPIRVBuiltinVariableKind *BuiltinKind = nullptr) const { SPIRVWord Kind; bool Found = hasDecorate(DecorationBuiltIn, 0, &Kind); if (!Found) return false; if (BuiltinKind) *BuiltinKind = static_cast(Kind); return true; } void setBuiltin(SPIRVBuiltinVariableKind Kind) { assert(isValid(Kind)); addDecorate(new SPIRVDecorate(DecorationBuiltIn, this, Kind)); } void setIsConstant(bool Is) { if (Is) addDecorate(new SPIRVDecorate(DecorationConstant, this)); else eraseDecorate(DecorationConstant); } std::vector getNonLiteralOperands() const override { if (SPIRVValue *V = getInitializer()) return std::vector(1, V); return std::vector(); } protected: void validate() const override { SPIRVValue::validate(); assert(isValid(StorageClass)); assert(Initializer.size() == 1 || Initializer.empty()); assert(getType()->isTypePointer()); } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Initializer.resize(WordCount - 4); } _SPIRV_DEF_ENCDEC4(Type, Id, StorageClass, Initializer) SPIRVStorageClassKind StorageClass; std::vector Initializer; }; class SPIRVStore : public SPIRVInstruction, public SPIRVMemoryAccess { public: const static SPIRVWord FixedWords = 3; // Complete constructor SPIRVStore(SPIRVId PointerId, SPIRVId ValueId, const std::vector &TheMemoryAccess, SPIRVBasicBlock *TheBB) : SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OpStore, TheBB), SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess), PtrId(PointerId), ValId(ValueId) { setAttr(); validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVStore() : SPIRVInstruction(OpStore), SPIRVMemoryAccess(), PtrId(SPIRVID_INVALID), ValId(SPIRVID_INVALID) { setAttr(); } SPIRVValue *getSrc() const { return getValue(ValId); } SPIRVValue *getDst() const { return getValue(PtrId); } protected: void setAttr() { setHasNoType(); setHasNoId(); } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); MemoryAccess.resize(TheWordCount - FixedWords); } void encode(spv_ostream &O) const override { getEncoder(O) << PtrId << ValId << MemoryAccess; } void decode(std::istream &I) override { getDecoder(I) >> PtrId >> ValId >> MemoryAccess; memoryAccessUpdate(MemoryAccess); } void validate() const override { SPIRVInstruction::validate(); if (getSrc()->isForward() || getDst()->isForward()) return; assert(getValueType(PtrId)->getPointerElementType() == getValueType(ValId) && "Inconsistent operand types"); } private: std::vector MemoryAccess; SPIRVId PtrId; SPIRVId ValId; }; class SPIRVLoad : public SPIRVInstruction, public SPIRVMemoryAccess { public: const static SPIRVWord FixedWords = 4; // Complete constructor SPIRVLoad(SPIRVId TheId, SPIRVId PointerId, const std::vector &TheMemoryAccess, SPIRVBasicBlock *TheBB) : SPIRVInstruction( FixedWords + TheMemoryAccess.size(), OpLoad, TheBB->getValueType(PointerId)->getPointerElementType(), TheId, TheBB), SPIRVMemoryAccess(TheMemoryAccess), PtrId(PointerId), MemoryAccess(TheMemoryAccess) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVLoad() : SPIRVInstruction(OpLoad), SPIRVMemoryAccess(), PtrId(SPIRVID_INVALID) {} SPIRVValue *getSrc() const { return Module->get(PtrId); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); MemoryAccess.resize(TheWordCount - FixedWords); } void encode(spv_ostream &O) const override { getEncoder(O) << Type << Id << PtrId << MemoryAccess; } void decode(std::istream &I) override { getDecoder(I) >> Type >> Id >> PtrId >> MemoryAccess; memoryAccessUpdate(MemoryAccess); } void validate() const override { SPIRVInstruction::validate(); assert((getValue(PtrId)->isForward() || Type == getValueType(PtrId)->getPointerElementType()) && "Inconsistent types"); } private: SPIRVId PtrId; std::vector MemoryAccess; }; class SPIRVBinary : public SPIRVInstTemplateBase { protected: void validate() const override { SPIRVId Op1 = Ops[0]; SPIRVId Op2 = Ops[1]; SPIRVType *Op1Ty, *Op2Ty; SPIRVInstruction::validate(); if (getValue(Op1)->isForward() || getValue(Op2)->isForward()) return; if (getValueType(Op1)->isTypeVector()) { Op1Ty = getValueType(Op1)->getVectorComponentType(); Op2Ty = getValueType(Op2)->getVectorComponentType(); assert(getValueType(Op1)->getVectorComponentCount() == getValueType(Op2)->getVectorComponentCount() && "Inconsistent Vector component width"); } else if (getValueType(Op1)->isTypeCooperativeMatrixKHR()) { Op1Ty = getValueType(Op1)->getVectorComponentType(); Op2Ty = getValueType(Op2)->getVectorComponentType(); assert(Op1Ty == Op2Ty && "Inconsistent Cooperative matrix types"); } else { Op1Ty = getValueType(Op1); Op2Ty = getValueType(Op2); } (void)Op1Ty; (void)Op2Ty; if (isBinaryOpCode(OpCode)) { assert(getValueType(Op1) == getValueType(Op2) && "Invalid type for binary instruction"); assert((Op1Ty->isTypeInt() || Op2Ty->isTypeFloat()) && "Invalid type for Binary instruction"); assert((Op1Ty->getBitWidth() == Op2Ty->getBitWidth()) && "Inconsistent BitWidth"); } else if (isShiftOpCode(OpCode)) { assert((Op1Ty->isTypeInt() || Op2Ty->isTypeInt()) && "Invalid type for shift instruction"); } else if (isLogicalOpCode(OpCode)) { assert((Op1Ty->isTypeBool() || Op2Ty->isTypeBool()) && "Invalid type for logical instruction"); } else if (isBitwiseOpCode(OpCode)) { assert((Op1Ty->isTypeInt() || Op2Ty->isTypeInt()) && "Invalid type for bitwise instruction"); assert((Op1Ty->getIntegerBitWidth() == Op2Ty->getIntegerBitWidth()) && "Inconsistent BitWidth"); } else if (isBinaryPtrOpCode(OpCode)) { assert((Op1Ty->isTypePointer() && Op2Ty->isTypePointer()) && "Invalid types for PtrEqual, PtrNotEqual, or PtrDiff instruction"); assert(static_cast(Op1Ty)->getElementType() == static_cast(Op2Ty)->getElementType() && "Invalid types for PtrEqual, PtrNotEqual, or PtrDiff instruction"); } else { assert(0 && "Invalid op code!"); } } SPIRVWord getRequiredSPIRVVersion() const override { if (isBinaryPtrOpCode(OpCode)) return static_cast(VersionNumber::SPIRV_1_4); return static_cast(VersionNumber::SPIRV_1_0); } SPIRVCapVec getRequiredCapability() const override { if (OpCode == OpDot) { const SPIRVType *OpTy = getValueType(Ops[0]); if (OpTy && OpTy->isTypeVector()) { OpTy = OpTy->getVectorComponentType(); if (OpTy && OpTy->isTypeFloat(16, FPEncodingBFloat16KHR)) { return getVec(CapabilityBFloat16DotProductKHR); } } } return SPIRVInstruction::getRequiredCapability(); } }; template class SPIRVBinaryInst : public SPIRVInstTemplate {}; #define _SPIRV_OP(x) typedef SPIRVBinaryInst SPIRV##x; _SPIRV_OP(IAdd) _SPIRV_OP(IAddCarry) _SPIRV_OP(FAdd) _SPIRV_OP(ISub) _SPIRV_OP(ISubBorrow) _SPIRV_OP(FSub) _SPIRV_OP(IMul) _SPIRV_OP(FMul) _SPIRV_OP(SMulExtended) _SPIRV_OP(UMulExtended) _SPIRV_OP(UDiv) _SPIRV_OP(SDiv) _SPIRV_OP(FDiv) _SPIRV_OP(SRem) _SPIRV_OP(SMod) _SPIRV_OP(FRem) _SPIRV_OP(FMod) _SPIRV_OP(UMod) _SPIRV_OP(ShiftLeftLogical) _SPIRV_OP(ShiftRightLogical) _SPIRV_OP(ShiftRightArithmetic) _SPIRV_OP(LogicalAnd) _SPIRV_OP(LogicalOr) _SPIRV_OP(LogicalEqual) _SPIRV_OP(LogicalNotEqual) _SPIRV_OP(BitwiseAnd) _SPIRV_OP(BitwiseOr) _SPIRV_OP(BitwiseXor) _SPIRV_OP(Dot) _SPIRV_OP(PtrEqual) _SPIRV_OP(PtrNotEqual) _SPIRV_OP(PtrDiff) #undef _SPIRV_OP template class SPIRVInstNoOperand : public SPIRVInstruction { public: // Complete constructor SPIRVInstNoOperand(SPIRVBasicBlock *TheBB) : SPIRVInstruction(1, TheOpCode, TheBB) { setAttr(); validate(); } // Incomplete constructor SPIRVInstNoOperand() : SPIRVInstruction(TheOpCode) { setAttr(); } protected: void setAttr() { setHasNoId(); setHasNoType(); } _SPIRV_DEF_ENCDEC0 }; typedef SPIRVInstNoOperand SPIRVReturn; typedef SPIRVInstNoOperand SPIRVUnreachable; class SPIRVReturnValue : public SPIRVInstruction { public: static const Op OC = OpReturnValue; // Complete constructor SPIRVReturnValue(SPIRVValue *TheReturnValue, SPIRVBasicBlock *TheBB) : SPIRVInstruction(2, OC, TheBB), ReturnValueId(TheReturnValue->getId()) { setAttr(); validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVReturnValue() : SPIRVInstruction(OC), ReturnValueId(SPIRVID_INVALID) { setAttr(); } SPIRVValue *getReturnValue() const { return getValue(ReturnValueId); } protected: void setAttr() { setHasNoId(); setHasNoType(); } _SPIRV_DEF_ENCDEC1(ReturnValueId) void validate() const override { SPIRVInstruction::validate(); } SPIRVId ReturnValueId; }; class SPIRVBranch : public SPIRVInstruction { public: static const Op OC = OpBranch; // Complete constructor SPIRVBranch(SPIRVLabel *TheTargetLabel, SPIRVBasicBlock *TheBB) : SPIRVInstruction(2, OC, TheBB), TargetLabelId(TheTargetLabel->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVBranch() : SPIRVInstruction(OC), TargetLabelId(SPIRVID_INVALID) { setHasNoId(); setHasNoType(); } SPIRVValue *getTargetLabel() const { return getValue(TargetLabelId); } protected: _SPIRV_DEF_ENCDEC1(TargetLabelId) void validate() const override { SPIRVInstruction::validate(); assert(WordCount == 2); assert(OpCode == OC); assert(getTargetLabel()->isLabel() || getTargetLabel()->isForward()); } SPIRVId TargetLabelId; }; class SPIRVBranchConditional : public SPIRVInstruction { public: static const Op OC = OpBranchConditional; // Complete constructor SPIRVBranchConditional(SPIRVValue *TheCondition, SPIRVLabel *TheTrueLabel, SPIRVLabel *TheFalseLabel, SPIRVBasicBlock *TheBB) : SPIRVInstruction(4, OC, TheBB), ConditionId(TheCondition->getId()), TrueLabelId(TheTrueLabel->getId()), FalseLabelId(TheFalseLabel->getId()) { validate(); } SPIRVBranchConditional(SPIRVValue *TheCondition, SPIRVLabel *TheTrueLabel, SPIRVLabel *TheFalseLabel, SPIRVBasicBlock *TheBB, SPIRVWord TrueWeight, SPIRVWord FalseWeight) : SPIRVInstruction(6, OC, TheBB), ConditionId(TheCondition->getId()), TrueLabelId(TheTrueLabel->getId()), FalseLabelId(TheFalseLabel->getId()) { BranchWeights.push_back(TrueWeight); BranchWeights.push_back(FalseWeight); validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVBranchConditional() : SPIRVInstruction(OC), ConditionId(SPIRVID_INVALID), TrueLabelId(SPIRVID_INVALID), FalseLabelId(SPIRVID_INVALID) { setHasNoId(); setHasNoType(); } SPIRVValue *getCondition() const { return getValue(ConditionId); } SPIRVLabel *getTrueLabel() const { return get(TrueLabelId); } SPIRVLabel *getFalseLabel() const { return get(FalseLabelId); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); BranchWeights.resize(TheWordCount - 4); } _SPIRV_DEF_ENCDEC4(ConditionId, TrueLabelId, FalseLabelId, BranchWeights) void validate() const override { SPIRVInstruction::validate(); assert(WordCount == 4 || WordCount == 6); assert(WordCount == BranchWeights.size() + 4); assert(OpCode == OC); assert(getCondition()->isForward() || getCondition()->getType()->isTypeBool()); assert(getTrueLabel()->isForward() || getTrueLabel()->isLabel()); assert(getFalseLabel()->isForward() || getFalseLabel()->isLabel()); if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) assert(TrueLabelId != FalseLabelId); } SPIRVId ConditionId; SPIRVId TrueLabelId; SPIRVId FalseLabelId; std::vector BranchWeights; }; class SPIRVPhi : public SPIRVInstruction { public: static const Op OC = OpPhi; static const SPIRVWord FixedWordCount = 3; SPIRVPhi(SPIRVType *TheType, SPIRVId TheId, const std::vector &ThePairs, SPIRVBasicBlock *BB) : SPIRVInstruction(ThePairs.size() + FixedWordCount, OC, TheType, TheId, BB) { Pairs = getIds(ThePairs); validate(); assert(BB && "Invalid BB"); } SPIRVPhi() : SPIRVInstruction(OC) {} std::vector getPairs() { return getValues(Pairs); } void addPair(SPIRVValue *Value, SPIRVBasicBlock *BB) { Pairs.push_back(Value->getId()); Pairs.push_back(BB->getId()); WordCount = Pairs.size() + FixedWordCount; validate(); } void setPairs(const std::vector &ThePairs) { Pairs = getIds(ThePairs); WordCount = Pairs.size() + FixedWordCount; validate(); } void foreachPair( std::function Func) { for (size_t I = 0, E = Pairs.size() / 2; I != E; ++I) { SPIRVEntry *Value, *BB; if (!Module->exist(Pairs[2 * I], &Value) || !Module->exist(Pairs[2 * I + 1], &BB)) continue; Func(static_cast(Value), static_cast(BB), I); } } void foreachPair(std::function Func) const { for (size_t I = 0, E = Pairs.size() / 2; I != E; ++I) { SPIRVEntry *Value, *BB; if (!Module->exist(Pairs[2 * I], &Value) || !Module->exist(Pairs[2 * I + 1], &BB)) continue; Func(static_cast(Value), static_cast(BB)); } } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Pairs.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC3(Type, Id, Pairs) void validate() const override { assert(WordCount == Pairs.size() + FixedWordCount); assert(OpCode == OC); assert(Pairs.size() % 2 == 0); foreachPair([=](SPIRVValue *IncomingV, SPIRVBasicBlock *IncomingBB) { assert(IncomingV->isForward() || IncomingV->getType() == Type); assert(IncomingBB->isBasicBlock() || IncomingBB->isForward()); }); SPIRVInstruction::validate(); } protected: std::vector Pairs; }; class SPIRVFPGARegINTELInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityFPGARegINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_fpga_reg; } protected: void validate() const override { SPIRVInstruction::validate(); assert(OpCode == OpFPGARegINTEL && "Invalid op code for FPGARegINTEL instruction"); assert(getType() == getValueType(Ops[0]) && "Inconsistent type"); } }; typedef SPIRVInstTemplate SPIRVFPGARegINTEL; class SPIRVCompare : public SPIRVInstTemplateBase { protected: void validate() const override { auto Op1 = Ops[0]; auto Op2 = Ops[1]; SPIRVType *Op1Ty, *Op2Ty, *ResTy; SPIRVInstruction::validate(); if (getValue(Op1)->isForward() || getValue(Op2)->isForward()) return; (void)Op1Ty; (void)Op2Ty; (void)ResTy; if (getValueType(Op1)->isTypeVector()) { Op1Ty = getValueType(Op1)->getVectorComponentType(); Op2Ty = getValueType(Op2)->getVectorComponentType(); ResTy = Type->getVectorComponentType(); assert(getValueType(Op1)->getVectorComponentCount() == getValueType(Op2)->getVectorComponentCount() && "Inconsistent Vector component width"); } else { Op1Ty = getValueType(Op1); Op2Ty = getValueType(Op2); ResTy = Type; } assert(isCmpOpCode(OpCode) && "Invalid op code for cmp inst"); if (OpCode == OpLessOrGreater) assert(this->getModule()->getSPIRVVersion() <= static_cast(VersionNumber::SPIRV_1_5) && "OpLessOrGreater is removed starting from SPIR-V 1.6"); assert((ResTy->isTypeBool() || ResTy->isTypeInt()) && "Invalid type for compare instruction"); assert(Op1Ty == Op2Ty && "Inconsistent types"); } }; template class SPIRVCmpInst : public SPIRVInstTemplate {}; #define _SPIRV_OP(x) typedef SPIRVCmpInst SPIRV##x; _SPIRV_OP(IEqual) _SPIRV_OP(FOrdEqual) _SPIRV_OP(FUnordEqual) _SPIRV_OP(INotEqual) _SPIRV_OP(FOrdNotEqual) _SPIRV_OP(FUnordNotEqual) _SPIRV_OP(ULessThan) _SPIRV_OP(SLessThan) _SPIRV_OP(FOrdLessThan) _SPIRV_OP(FUnordLessThan) _SPIRV_OP(UGreaterThan) _SPIRV_OP(SGreaterThan) _SPIRV_OP(FOrdGreaterThan) _SPIRV_OP(FUnordGreaterThan) _SPIRV_OP(ULessThanEqual) _SPIRV_OP(SLessThanEqual) _SPIRV_OP(FOrdLessThanEqual) _SPIRV_OP(FUnordLessThanEqual) _SPIRV_OP(UGreaterThanEqual) _SPIRV_OP(SGreaterThanEqual) _SPIRV_OP(FOrdGreaterThanEqual) _SPIRV_OP(FUnordGreaterThanEqual) _SPIRV_OP(LessOrGreater) _SPIRV_OP(Ordered) _SPIRV_OP(Unordered) #undef _SPIRV_OP class SPIRVSelectBase : public SPIRVInstTemplateBase { public: SPIRVValue *getCondition() { return getValue(Ops[0]); } SPIRVValue *getTrueValue() { return getValue(Ops[1]); } SPIRVValue *getFalseValue() { return getValue(Ops[2]); } protected: void validate() const override { SPIRVId Condition = Ops[0]; SPIRVId Op1 = Ops[1]; SPIRVId Op2 = Ops[2]; SPIRVInstruction::validate(); if (getValue(Condition)->isForward() || getValue(Op1)->isForward() || getValue(Op2)->isForward()) return; SPIRVType *ConTy = getValueType(Condition)->isTypeVector() ? getValueType(Condition)->getVectorComponentType() : getValueType(Condition); (void)ConTy; assert(ConTy->isTypeBool() && "Invalid type"); assert(getType() == getValueType(Op1) && getType() == getValueType(Op2) && "Inconsistent type"); } }; typedef SPIRVInstTemplate SPIRVSelect; class SPIRVSelectionMerge : public SPIRVInstruction { public: static const Op OC = OpSelectionMerge; static const SPIRVWord FixedWordCount = 3; SPIRVSelectionMerge(SPIRVId TheMergeBlock, SPIRVWord TheSelectionControl, SPIRVBasicBlock *BB) : SPIRVInstruction(3, OC, BB), MergeBlock(TheMergeBlock), SelectionControl(TheSelectionControl) { validate(); assert(BB && "Invalid BB"); } SPIRVSelectionMerge() : SPIRVInstruction(OC), MergeBlock(SPIRVID_INVALID), SelectionControl(SPIRVWORD_MAX) { setHasNoId(); setHasNoType(); } SPIRVId getMergeBlock() { return MergeBlock; } SPIRVWord getSelectionControl() { return SelectionControl; } _SPIRV_DEF_ENCDEC2(MergeBlock, SelectionControl) protected: SPIRVId MergeBlock; SPIRVWord SelectionControl; }; class SPIRVLoopMerge : public SPIRVInstruction { public: static const Op OC = OpLoopMerge; static const SPIRVWord FixedWordCount = 4; SPIRVLoopMerge(SPIRVId TheMergeBlock, SPIRVId TheContinueTarget, SPIRVWord TheLoopControl, std::vector TheLoopControlParameters, SPIRVBasicBlock *BB) : SPIRVInstruction(FixedWordCount + TheLoopControlParameters.size(), OC, BB), MergeBlock(TheMergeBlock), ContinueTarget(TheContinueTarget), LoopControl(TheLoopControl), LoopControlParameters(TheLoopControlParameters) { validate(); assert(BB && "Invalid BB"); } SPIRVLoopMerge() : SPIRVInstruction(OC), MergeBlock(SPIRVID_MAX), LoopControl(SPIRVWORD_MAX) { setHasNoId(); setHasNoType(); } SPIRVId getMergeBlock() { return MergeBlock; } SPIRVId getContinueTarget() { return ContinueTarget; } SPIRVWord getLoopControl() const { return LoopControl; } std::vector getLoopControlParameters() const { return LoopControlParameters; } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); LoopControlParameters.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC4(MergeBlock, ContinueTarget, LoopControl, LoopControlParameters) protected: SPIRVId MergeBlock; SPIRVId ContinueTarget; SPIRVWord LoopControl; std::vector LoopControlParameters; }; class SPIRVSwitch : public SPIRVInstruction { public: static const Op OC = OpSwitch; static const SPIRVWord FixedWordCount = 3; typedef std::vector LiteralTy; typedef std::pair PairTy; SPIRVSwitch(SPIRVValue *TheSelect, SPIRVBasicBlock *TheDefault, const std::vector &ThePairs, SPIRVBasicBlock *BB) : SPIRVInstruction(FixedWordCount, OC, BB), Select(TheSelect->getId()), Default(TheDefault->getId()) { if (!ThePairs.empty()) SPIRVEntry::setWordCount( ThePairs.size() * (ThePairs.at(0).first.size() + 1) + FixedWordCount); for (auto &I : ThePairs) { for (auto &U : I.first) Pairs.push_back(U); Pairs.push_back(I.second->getId()); } validate(); assert(BB && "Invalid BB"); } SPIRVSwitch() : SPIRVInstruction(OC), Select(SPIRVWORD_MAX), Default(SPIRVWORD_MAX) { setHasNoId(); setHasNoType(); } std::vector getPairs() const { return getValues(Pairs); } SPIRVValue *getSelect() const { return getValue(Select); } SPIRVBasicBlock *getDefault() const { return static_cast(getValue(Default)); } size_t getLiteralSize() const { unsigned ByteWidth = getSelect()->getType()->getBitWidth() / 8; unsigned Remainder = (ByteWidth % sizeof(SPIRVWord)) != 0; return (ByteWidth / sizeof(SPIRVWord)) + Remainder; } size_t getPairSize() const { return getLiteralSize() + 1; } size_t getNumPairs() const { return Pairs.size() / getPairSize(); } void foreachPair(std::function Func) const { unsigned PairSize = getPairSize(); for (size_t I = 0, E = getNumPairs(); I != E; ++I) { SPIRVEntry *BB; LiteralTy Literals; if (!Module->exist(Pairs[PairSize * I + getLiteralSize()], &BB)) continue; for (size_t J = 0; J < getLiteralSize(); ++J) { Literals.push_back(Pairs.at(PairSize * I + J)); } Func(Literals, static_cast(BB)); } } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Pairs.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC3(Select, Default, Pairs) void validate() const override { assert(WordCount == Pairs.size() + FixedWordCount); assert(OpCode == OC); assert(Pairs.size() % getPairSize() == 0); foreachPair([=](LiteralTy Literals, SPIRVBasicBlock *BB) { assert(BB->isBasicBlock() || BB->isForward()); }); SPIRVInstruction::validate(); } protected: SPIRVId Select; SPIRVId Default; std::vector Pairs; }; class SPIRVVectorTimesScalar : public SPIRVInstruction { public: static const Op OC = OpVectorTimesScalar; static const SPIRVWord FixedWordCount = 4; // Complete constructor SPIRVVectorTimesScalar(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheVector, SPIRVId TheScalar, SPIRVBasicBlock *BB) : SPIRVInstruction(5, OC, TheType, TheId, BB), Vector(TheVector), Scalar(TheScalar) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVVectorTimesScalar() : SPIRVInstruction(OC), Vector(SPIRVID_INVALID), Scalar(SPIRVID_INVALID) { } SPIRVValue *getVector() const { return getValue(Vector); } SPIRVValue *getScalar() const { return getValue(Scalar); } std::vector getOperands() override { std::vector Operands; Operands.push_back(Vector); Operands.push_back(Scalar); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC4(Type, Id, Vector, Scalar) void validate() const override { SPIRVInstruction::validate(); if (getValue(Vector)->isForward() || getValue(Scalar)->isForward()) return; assert(getValueType(Vector)->isTypeVector() && getValueType(Vector)->getVectorComponentType()->isTypeFloat() && "First operand must be a vector of floating-point type"); assert(getValueType(getId())->isTypeVector() && getValueType(getId())->getVectorComponentType()->isTypeFloat() && "Result type must be a vector of floating-point type"); assert( getValueType(Vector)->getVectorComponentType() == getValueType(getId())->getVectorComponentType() && "Scalar must have the same type as the Component Type in Result Type"); SPIRVInstruction::validate(); } protected: SPIRVId Vector; SPIRVId Scalar; }; class SPIRVVectorTimesMatrix : public SPIRVInstruction { public: static const Op OC = OpVectorTimesMatrix; static const SPIRVWord FixedWordCount = 4; // Complete constructor SPIRVVectorTimesMatrix(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheVector, SPIRVId TheMatrix, SPIRVBasicBlock *BB) : SPIRVInstruction(5, OC, TheType, TheId, BB), Vector(TheVector), Matrix(TheMatrix) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVVectorTimesMatrix() : SPIRVInstruction(OC), Vector(SPIRVID_INVALID), Matrix(SPIRVID_INVALID) { } SPIRVValue *getVector() const { return getValue(Vector); } SPIRVValue *getMatrix() const { return getValue(Matrix); } std::vector getOperands() override { std::vector Operands; Operands.push_back(Vector); Operands.push_back(Matrix); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC4(Type, Id, Vector, Matrix) void validate() const override { SPIRVInstruction::validate(); if (getValue(Vector)->isForward() || getValue(Matrix)->isForward()) return; SPIRVType *Ty = getType()->getScalarType(); SPIRVType *MTy = getValueType(Matrix)->getScalarType(); SPIRVType *VTy = getValueType(Vector)->getScalarType(); (void)Ty; (void)MTy; (void)VTy; assert(Ty->isTypeFloat() && "Invalid result type for OpVectorTimesMatrix"); assert(VTy->isTypeFloat() && "Invalid Vector type for OpVectorTimesMatrix"); assert(MTy->isTypeFloat() && "Invalid Matrix type for OpVectorTimesMatrix"); assert(Ty == MTy && Ty == VTy && "Mismatch float type"); } private: SPIRVId Vector; SPIRVId Matrix; }; class SPIRVMatrixTimesScalar : public SPIRVInstruction { public: static const Op OC = OpMatrixTimesScalar; static const SPIRVWord FixedWordCount = 4; // Complete constructor SPIRVMatrixTimesScalar(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix, SPIRVId TheScalar, SPIRVBasicBlock *BB) : SPIRVInstruction(5, OC, TheType, TheId, BB), Matrix(TheMatrix), Scalar(TheScalar) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVMatrixTimesScalar() : SPIRVInstruction(OC), Matrix(SPIRVID_INVALID), Scalar(SPIRVID_INVALID) { } SPIRVValue *getMatrix() const { return getValue(Matrix); } SPIRVValue *getScalar() const { return getValue(Scalar); } std::vector getOperands() override { std::vector Operands; Operands.push_back(Matrix); Operands.push_back(Scalar); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC4(Type, Id, Matrix, Scalar) void validate() const override { SPIRVInstruction::validate(); if (getValue(Matrix)->isForward() || getValue(Scalar)->isForward()) return; SPIRVType *Ty = getType()->getScalarType(); SPIRVType *MTy = getValueType(Matrix)->getScalarType(); SPIRVType *STy = getValueType(Scalar); (void)Ty; (void)MTy; (void)STy; assert(Ty && Ty->isTypeFloat() && "Invalid result type for OpMatrixTimesScalar"); assert(MTy && MTy->isTypeFloat() && "Invalid Matrix type for OpMatrixTimesScalar"); assert(STy->isTypeFloat() && "Invalid Scalar type for OpMatrixTimesScalar"); assert(Ty == MTy && Ty == STy && "Mismatch float type"); } private: SPIRVId Matrix; SPIRVId Scalar; }; class SPIRVMatrixTimesVector : public SPIRVInstruction { public: static const Op OC = OpMatrixTimesVector; static const SPIRVWord FixedWordCount = 4; // Complete constructor SPIRVMatrixTimesVector(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix, SPIRVId TheVector, SPIRVBasicBlock *BB) : SPIRVInstruction(5, OC, TheType, TheId, BB), Matrix(TheMatrix), Vector(TheVector) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVMatrixTimesVector() : SPIRVInstruction(OC), Matrix(SPIRVID_INVALID), Vector(SPIRVID_INVALID) { } SPIRVValue *getMatrix() const { return getValue(Matrix); } SPIRVValue *getVector() const { return getValue(Vector); } std::vector getOperands() override { std::vector Operands; Operands.push_back(Matrix); Operands.push_back(Vector); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC4(Type, Id, Matrix, Vector) void validate() const override { SPIRVInstruction::validate(); if (getValue(Matrix)->isForward() || getValue(Vector)->isForward()) return; SPIRVType *Ty = getType()->getScalarType(); SPIRVType *MTy = getValueType(Matrix)->getScalarType(); SPIRVType *VTy = getValueType(Vector)->getScalarType(); (void)Ty; (void)MTy; (void)VTy; assert(Ty->isTypeFloat() && "Invalid result type for OpMatrixTimesVector"); assert(MTy->isTypeFloat() && "Invalid Matrix type for OpMatrixTimesVector"); assert(VTy->isTypeFloat() && "Invalid Vector type for OpMatrixTimesVector"); assert(Ty == MTy && Ty == VTy && "Mismatch float type"); } private: SPIRVId Matrix; SPIRVId Vector; }; class SPIRVMatrixTimesMatrix : public SPIRVInstruction { public: static const Op OC = OpMatrixTimesMatrix; static const SPIRVWord FixedWordCount = 4; // Complete constructor SPIRVMatrixTimesMatrix(SPIRVType *TheType, SPIRVId TheId, SPIRVId M1, SPIRVId M2, SPIRVBasicBlock *BB) : SPIRVInstruction(5, OC, TheType, TheId, BB), LeftMatrix(M1), RightMatrix(M2) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVMatrixTimesMatrix() : SPIRVInstruction(OC), LeftMatrix(SPIRVID_INVALID), RightMatrix(SPIRVID_INVALID) {} SPIRVValue *getLeftMatrix() const { return getValue(LeftMatrix); } SPIRVValue *getRightMatrix() const { return getValue(RightMatrix); } std::vector getOperands() override { std::vector Operands; Operands.push_back(LeftMatrix); Operands.push_back(RightMatrix); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC4(Type, Id, LeftMatrix, RightMatrix) void validate() const override { SPIRVInstruction::validate(); if (getValue(LeftMatrix)->isForward() || getValue(RightMatrix)->isForward()) return; SPIRVType *Ty = getType()->getScalarType(); SPIRVType *LMTy = getValueType(LeftMatrix)->getScalarType(); SPIRVType *RMTy = getValueType(RightMatrix)->getScalarType(); (void)Ty; (void)LMTy; (void)RMTy; assert(Ty->isTypeFloat() && "Invalid result type for OpMatrixTimesMatrix"); assert(LMTy->isTypeFloat() && "Invalid Matrix type for OpMatrixTimesMatrix"); assert(RMTy->isTypeFloat() && "Invalid Matrix type for OpMatrixTimesMatrix"); assert(Ty == LMTy && Ty == RMTy && "Mismatch float type"); } private: SPIRVId LeftMatrix; SPIRVId RightMatrix; }; class SPIRVTranspose : public SPIRVInstruction { public: static const Op OC = OpTranspose; static const SPIRVWord FixedWordCount = 3; // Complete constructor SPIRVTranspose(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheMatrix, SPIRVBasicBlock *BB) : SPIRVInstruction(4, OC, TheType, TheId, BB), Matrix(TheMatrix) { validate(); assert(BB && "Invalid BB"); } // Incomplete constructor SPIRVTranspose() : SPIRVInstruction(OC), Matrix(SPIRVID_INVALID) {} SPIRVValue *getMatrix() const { return getValue(Matrix); } std::vector getOperands() override { std::vector Operands; Operands.push_back(Matrix); return getValues(Operands); } void setWordCount(SPIRVWord FixedWordCount) override { SPIRVEntry::setWordCount(FixedWordCount); } _SPIRV_DEF_ENCDEC3(Type, Id, Matrix) void validate() const override { SPIRVInstruction::validate(); if (getValue(Matrix)->isForward()) return; SPIRVType *Ty = getType()->getScalarType(); SPIRVType *MTy = getValueType(Matrix)->getScalarType(); (void)Ty; (void)MTy; assert(Ty->isTypeFloat() && "Invalid result type for OpTranspose"); assert(Ty == MTy && "Mismatch float type"); } private: SPIRVId Matrix; }; class SPIRVSizeOfInstBase : public SPIRVInstTemplateBase { protected: SPIRVWord getRequiredSPIRVVersion() const override { return static_cast(VersionNumber::SPIRV_1_1); } }; typedef SPIRVInstTemplate SPIRVSizeOf; class SPIRVUnary : public SPIRVInstTemplateBase { protected: void validate() const override { auto Op = Ops[0]; SPIRVInstruction::validate(); if (getValue(Op)->isForward()) return; if (isGenericNegateOpCode(OpCode)) { SPIRVType *ResTy = Type->isTypeVector() || Type->isTypeCooperativeMatrixKHR() ? Type->getVectorComponentType() : Type; SPIRVType *OpTy = Type->isTypeVector() || Type->isTypeCooperativeMatrixKHR() ? getValueType(Op)->getVectorComponentType() : getValueType(Op); (void)ResTy; (void)OpTy; assert(getType() == getValueType(Op) && "Inconsistent type"); assert((ResTy->isTypeInt() || ResTy->isTypeFloat()) && "Invalid type for Generic Negate instruction"); assert((ResTy->getBitWidth() == OpTy->getBitWidth()) && "Invalid bitwidth for Generic Negate instruction"); assert((Type->isTypeVector() ? (Type->getVectorComponentCount() == getValueType(Op)->getVectorComponentCount()) : 1) && "Invalid vector component Width for Generic Negate instruction"); } } }; template class SPIRVUnaryInst : public SPIRVInstTemplate {}; #define _SPIRV_OP(x) typedef SPIRVUnaryInst SPIRV##x; _SPIRV_OP(ConvertFToU) _SPIRV_OP(ConvertFToS) _SPIRV_OP(ConvertSToF) _SPIRV_OP(ConvertUToF) _SPIRV_OP(UConvert) _SPIRV_OP(SConvert) _SPIRV_OP(FConvert) _SPIRV_OP(SatConvertSToU) _SPIRV_OP(SatConvertUToS) _SPIRV_OP(ConvertPtrToU) _SPIRV_OP(ConvertUToPtr) _SPIRV_OP(PtrCastToGeneric) _SPIRV_OP(GenericCastToPtr) _SPIRV_OP(CrossWorkgroupCastToPtrINTEL) _SPIRV_OP(PtrCastToCrossWorkgroupINTEL) _SPIRV_OP(Bitcast) _SPIRV_OP(SNegate) _SPIRV_OP(FNegate) _SPIRV_OP(Not) _SPIRV_OP(LogicalNot) _SPIRV_OP(IsNan) _SPIRV_OP(IsInf) _SPIRV_OP(IsFinite) _SPIRV_OP(IsNormal) _SPIRV_OP(SignBitSet) _SPIRV_OP(Any) _SPIRV_OP(All) _SPIRV_OP(BitCount) _SPIRV_OP(ArithmeticFenceEXT) #undef _SPIRV_OP class SPIRVAccessChainBase : public SPIRVInstTemplateBase { public: SPIRVValue *getBase() { return this->getValue(this->Ops[0]); } std::vector getIndices() const { std::vector IndexWords(this->Ops.begin() + 1, this->Ops.end()); return this->getValues(IndexWords); } bool isInBounds() { return OpCode == OpInBoundsAccessChain || OpCode == OpInBoundsPtrAccessChain; } bool hasPtrIndex() { return OpCode == OpPtrAccessChain || OpCode == OpInBoundsPtrAccessChain; } }; template class SPIRVAccessChainGeneric : public SPIRVInstTemplate { }; typedef SPIRVAccessChainGeneric SPIRVAccessChain; typedef SPIRVAccessChainGeneric SPIRVInBoundsAccessChain; typedef SPIRVAccessChainGeneric SPIRVPtrAccessChain; typedef SPIRVAccessChainGeneric SPIRVInBoundsPtrAccessChain; class SPIRVLoopControlINTEL : public SPIRVInstruction { public: static const Op OC = OpLoopControlINTEL; static const SPIRVWord FixedWordCount = 2; SPIRVLoopControlINTEL(SPIRVWord TheLoopControl, std::vector TheLoopControlParameters, SPIRVBasicBlock *BB) : SPIRVInstruction(FixedWordCount + TheLoopControlParameters.size(), OC, BB), LoopControl(TheLoopControl), LoopControlParameters(TheLoopControlParameters) { validate(); assert(BB && "Invalid BB"); } SPIRVLoopControlINTEL() : SPIRVInstruction(OC), LoopControl(SPIRVWORD_MAX) { setHasNoId(); setHasNoType(); } SPIRVWord getLoopControl() const { return LoopControl; } std::vector getLoopControlParameters() const { return LoopControlParameters; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityUnstructuredLoopControlsINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_unstructured_loop_controls; } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); LoopControlParameters.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC2(LoopControl, LoopControlParameters) protected: SPIRVWord LoopControl; std::vector LoopControlParameters; }; template class SPIRVFunctionCallGeneric : public SPIRVInstruction { public: SPIRVFunctionCallGeneric(SPIRVType *TheType, SPIRVId TheId, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, TheId, BB), Args(TheArgs) { SPIRVFunctionCallGeneric::validate(); assert(BB && "Invalid BB"); } SPIRVFunctionCallGeneric(SPIRVType *TheType, SPIRVId TheId, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, TheId, BB) { Args = getIds(TheArgs); SPIRVFunctionCallGeneric::validate(); assert(BB && "Invalid BB"); } SPIRVFunctionCallGeneric(SPIRVModule *BM, SPIRVWord ResId, SPIRVType *TheType, const std::vector &TheArgs) : SPIRVInstruction(TheArgs.size() + FixedWordCount, OC, TheType, ResId, BM), Args(TheArgs) {} SPIRVFunctionCallGeneric() : SPIRVInstruction(OC) {} const std::vector &getArguments() const { return Args; } void setArguments(const std::vector &A) { Args = A; setWordCount(Args.size() + FixedWordCount); } std::vector getArgumentValues() { return getValues(Args); } std::vector getArgumentValueTypes() const { std::vector ArgTypes; for (auto &I : Args) ArgTypes.push_back(getValue(I)->getType()); return ArgTypes; } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Args.resize(TheWordCount - FixedWordCount); } void validate() const override { SPIRVInstruction::validate(); } protected: std::vector Args; }; class SPIRVFunctionCall : public SPIRVFunctionCallGeneric { public: SPIRVFunctionCall(SPIRVId TheId, SPIRVFunction *TheFunction, const std::vector &TheArgs, SPIRVBasicBlock *BB); SPIRVFunctionCall() : FunctionId(SPIRVID_INVALID) {} SPIRVFunction *getFunction() const { return get(FunctionId); } _SPIRV_DEF_ENCDEC4(Type, Id, FunctionId, Args) void validate() const override; bool isOperandLiteral(unsigned Index) const override { return false; } protected: SPIRVId FunctionId; }; class SPIRVFunctionPointerCallINTEL : public SPIRVFunctionCallGeneric { public: SPIRVFunctionPointerCallINTEL(SPIRVId TheId, SPIRVValue *TheCalledValue, SPIRVType *TheReturnType, const std::vector &TheArgs, SPIRVBasicBlock *BB); SPIRVFunctionPointerCallINTEL() : CalledValueId(SPIRVID_INVALID) {} SPIRVValue *getCalledValue() const { return get(CalledValueId); } _SPIRV_DEF_ENCDEC4(Type, Id, CalledValueId, Args) void validate() const override; bool isOperandLiteral(unsigned Index) const override { return false; } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_function_pointers; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityFunctionPointersINTEL); } protected: SPIRVId CalledValueId; }; class SPIRVExtInst : public SPIRVFunctionCallGeneric { public: SPIRVExtInst(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheBuiltinSet, SPIRVWord TheEntryPoint, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVFunctionCallGeneric(TheType, TheId, TheArgs, BB), ExtSetId(TheBuiltinSet), ExtOp(TheEntryPoint) { setExtSetKindById(); SPIRVExtInst::validate(); } SPIRVExtInst(SPIRVType *TheType, SPIRVId TheId, SPIRVId TheBuiltinSet, SPIRVWord TheEntryPoint, const std::vector &TheArgs, SPIRVBasicBlock *BB) : SPIRVFunctionCallGeneric(TheType, TheId, TheArgs, BB), ExtSetId(TheBuiltinSet), ExtOp(TheEntryPoint) { setExtSetKindById(); SPIRVExtInst::validate(); } SPIRVExtInst(SPIRVModule *BM, SPIRVId ResId, SPIRVType *TheType, SPIRVExtInstSetKind SetKind, SPIRVWord SetId, SPIRVWord InstId, const std::vector &Args) : SPIRVFunctionCallGeneric(BM, ResId, TheType, Args), ExtSetKind(SetKind), ExtSetId(SetId), ExtOp(InstId) {} SPIRVExtInst(SPIRVExtInstSetKind SetKind = SPIRVEIS_Count, unsigned ExtOC = SPIRVWORD_MAX) : ExtSetKind(SetKind), ExtSetId(SPIRVWORD_MAX), ExtOp(ExtOC) {} void setExtSetId(unsigned Set) { ExtSetId = Set; } void setExtOp(unsigned ExtOC) { ExtOp = ExtOC; } SPIRVId getExtSetId() const { return ExtSetId; } SPIRVWord getExtOp() const { return ExtOp; } SPIRVExtInstSetKind getExtSetKind() const { return ExtSetKind; } void addContinuedInstruction(SPIRVExtInst *Inst) { ContinuedInstructions.push_back(Inst); } std::vector getContinuedInstructions() { return ContinuedInstructions; } void setExtSetKindById() { assert(Module && "Invalid module"); ExtSetKind = Module->getBuiltinSet(ExtSetId); assert((ExtSetKind == SPIRVEIS_OpenCL || ExtSetKind == SPIRVEIS_Debug || ExtSetKind == SPIRVEIS_OpenCL_DebugInfo_100 || ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_200 || ExtSetKind == SPIRVEIS_NonSemantic_AuxData) && "not supported"); } void encode(spv_ostream &O) const override { getEncoder(O) << Type << Id << ExtSetId; switch (ExtSetKind) { case SPIRVEIS_OpenCL: getEncoder(O) << ExtOpOCL; break; case SPIRVEIS_Debug: case SPIRVEIS_OpenCL_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_200: getEncoder(O) << ExtOpDebug; break; case SPIRVEIS_NonSemantic_AuxData: getEncoder(O) << ExtOpNonSemanticAuxData; break; default: assert(0 && "not supported"); getEncoder(O) << ExtOp; } getEncoder(O) << Args; } void decode(std::istream &I) override { getDecoder(I) >> Type >> Id >> ExtSetId; setExtSetKindById(); switch (ExtSetKind) { case SPIRVEIS_OpenCL: getDecoder(I) >> ExtOpOCL; break; case SPIRVEIS_Debug: case SPIRVEIS_OpenCL_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_100: case SPIRVEIS_NonSemantic_Shader_DebugInfo_200: getDecoder(I) >> ExtOpDebug; break; case SPIRVEIS_NonSemantic_AuxData: getDecoder(I) >> ExtOpNonSemanticAuxData; break; default: assert(0 && "not supported"); getDecoder(I) >> ExtOp; } SPIRVDecoder Decoder = getDecoder(I); Decoder >> Args; if (ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || ExtSetKind == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) { if (getExtOp() == SPIRVDebug::Instruction::Source) { for (SPIRVEntry *E : Decoder.getSourceContinuedInstructions()) { addContinuedInstruction(static_cast(E)); } } } } void validate() const override { SPIRVFunctionCallGeneric::validate(); validateBuiltin(ExtSetId, ExtOp); } bool isOperandLiteral(unsigned Index) const override { assert(ExtSetKind == SPIRVEIS_OpenCL && "Unsupported extended instruction set"); auto EOC = static_cast(ExtOp); switch (EOC) { default: return false; case OpenCLLIB::Vloadn: case OpenCLLIB::Vload_halfn: case OpenCLLIB::Vloada_halfn: return Index == 2; case OpenCLLIB::Vstore_half_r: case OpenCLLIB::Vstore_halfn_r: case OpenCLLIB::Vstorea_halfn_r: return Index == 3; } } std::vector getArgValues() { std::vector VArgs; for (size_t I = 0; I < Args.size(); ++I) { if (isOperandLiteral(I)) VArgs.push_back(Module->getLiteralAsConstant(Args[I])); else VArgs.push_back(getValue(Args[I])); } return VArgs; } std::vector getArgTypes() { std::vector ArgTypes; auto VArgs = getArgValues(); for (auto VArg : VArgs) ArgTypes.push_back(VArg->getType()); return ArgTypes; } llvm::Optional getRequiredExtension() const override { if (SPIRVBuiltinSetNameMap::map(ExtSetKind).find("NonSemantic.") == 0 && !Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) return ExtensionID::SPV_KHR_non_semantic_info; return {}; } protected: SPIRVExtInstSetKind ExtSetKind; SPIRVId ExtSetId; union { SPIRVWord ExtOp; OCLExtOpKind ExtOpOCL; SPIRVDebugExtOpKind ExtOpDebug; NonSemanticAuxDataOpKind ExtOpNonSemanticAuxData; }; std::vector ContinuedInstructions; }; class SPIRVCompositeConstruct : public SPIRVInstruction { public: const static Op OC = OpCompositeConstruct; const static SPIRVWord FixedWordCount = 3; // Complete constructor SPIRVCompositeConstruct(SPIRVType *TheType, SPIRVId TheId, const std::vector &TheConstituents, SPIRVBasicBlock *TheBB) : SPIRVInstruction(TheConstituents.size() + FixedWordCount, OC, TheType, TheId, TheBB), Constituents(TheConstituents) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVCompositeConstruct() : SPIRVInstruction(OC) {} std::vector getOperands() override { return getValues(Constituents); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Constituents.resize(TheWordCount - FixedWordCount); } _SPIRV_DEF_ENCDEC3(Type, Id, Constituents) void validate() const override { SPIRVInstruction::validate(); size_t TypeOpCode = this->getType()->getOpCode(); switch (TypeOpCode) { case OpTypeVector: assert(Constituents.size() > 1 && "There must be at least two Constituent operands in vector"); break; case OpTypeArray: case OpTypeStruct: case internal::OpTypeJointMatrixINTEL: case OpTypeCooperativeMatrixKHR: break; default: assert(false && "Invalid type"); } } std::vector Constituents; }; class SPIRVCompositeExtractBase : public SPIRVInstTemplateBase { public: SPIRVValue *getComposite() { return getValue(Ops[0]); } const std::vector getIndices() const { return std::vector(Ops.begin() + 1, Ops.end()); } protected: // ToDo: validate the result type is consistent with the base type and indices // need to trace through the base type for struct types void validate() const override { SPIRVInstruction::validate(); assert(OpCode == OpCompositeExtract); SPIRVId Composite = Ops[0]; (void)Composite; assert(getValueType(Composite)->isTypeArray() || getValueType(Composite)->isTypeStruct() || getValueType(Composite)->isTypeVector()); } }; typedef SPIRVInstTemplate SPIRVCompositeExtract; class SPIRVCompositeInsertBase : public SPIRVInstTemplateBase { public: SPIRVValue *getObject() { return getValue(Ops[0]); } SPIRVValue *getComposite() { return getValue(Ops[1]); } const std::vector getIndices() const { return std::vector(Ops.begin() + 2, Ops.end()); } protected: // ToDo: validate the object type is consistent with the base type and indices // need to trace through the base type for struct types void validate() const override { SPIRVInstruction::validate(); assert(OpCode == OpCompositeInsert); SPIRVId Composite = Ops[1]; (void)Composite; assert(getValueType(Composite)->isTypeArray() || getValueType(Composite)->isTypeStruct() || getValueType(Composite)->isTypeVector()); assert(Type == getValueType(Composite)); } }; typedef SPIRVInstTemplate SPIRVCompositeInsert; class SPIRVCopyObject : public SPIRVInstruction { public: const static Op OC = OpCopyObject; // Complete constructor SPIRVCopyObject(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheOperand, SPIRVBasicBlock *TheBB) : SPIRVInstruction(4, OC, TheType, TheId, TheBB), Operand(TheOperand->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVCopyObject() : SPIRVInstruction(OC), Operand(SPIRVID_INVALID) {} SPIRVValue *getOperand() { return getValue(Operand); } protected: _SPIRV_DEF_ENCDEC3(Type, Id, Operand) void validate() const override { SPIRVInstruction::validate(); } SPIRVId Operand; }; class SPIRVCopyLogical : public SPIRVInstruction { public: const static Op OC = OpCopyLogical; // Complete constructor SPIRVCopyLogical(SPIRVType *TheType, SPIRVId TheId, SPIRVValue *TheOperand, SPIRVBasicBlock *TheBB) : SPIRVInstruction(4, OC, TheType, TheId, TheBB), Operand(TheOperand->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVCopyLogical() : SPIRVInstruction(OC), Operand(SPIRVID_INVALID) {} SPIRVValue *getOperand() { return getValue(Operand); } std::vector getOperands() override { return {getOperand()}; } protected: _SPIRV_DEF_ENCDEC3(Type, Id, Operand) void validate() const override { SPIRVInstruction::validate(); } SPIRVId Operand; }; class SPIRVCopyMemory : public SPIRVInstruction, public SPIRVMemoryAccess { public: const static Op OC = OpCopyMemory; const static SPIRVWord FixedWords = 3; // Complete constructor SPIRVCopyMemory(SPIRVValue *TheTarget, SPIRVValue *TheSource, const std::vector &TheMemoryAccess, SPIRVBasicBlock *TheBB) : SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OC, TheBB), SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess), Target(TheTarget->getId()), Source(TheSource->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVCopyMemory() : SPIRVInstruction(OC), SPIRVMemoryAccess(), Target(SPIRVID_INVALID), Source(SPIRVID_INVALID) { setHasNoId(); setHasNoType(); } SPIRVValue *getSource() { return getValue(Source); } SPIRVValue *getTarget() { return getValue(Target); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); MemoryAccess.resize(TheWordCount - FixedWords); } void encode(spv_ostream &O) const override { getEncoder(O) << Target << Source << MemoryAccess; } void decode(std::istream &I) override { getDecoder(I) >> Target >> Source >> MemoryAccess; memoryAccessUpdate(MemoryAccess); } void validate() const override { assert(getValueType(Target)->isTypePointer() && "Invalid Target type"); assert(getValueType(Source)->isTypePointer() && "Invalid Source type"); assert(!(getValueType(Target)->getPointerElementType()->isTypeVoid()) && "Invalid Target element type"); assert(!(getValueType(Source)->getPointerElementType()->isTypeVoid()) && "Invalid Source element type"); assert(getValueType(Target)->getPointerElementType() == getValueType(Source)->getPointerElementType() && "Mismatching Target and Source element types"); SPIRVInstruction::validate(); } std::vector MemoryAccess; SPIRVId Target; SPIRVId Source; }; class SPIRVCopyMemorySized : public SPIRVInstruction, public SPIRVMemoryAccess { public: const static Op OC = OpCopyMemorySized; const static SPIRVWord FixedWords = 4; // Complete constructor SPIRVCopyMemorySized(SPIRVValue *TheTarget, SPIRVValue *TheSource, SPIRVValue *TheSize, const std::vector &TheMemoryAccess, SPIRVBasicBlock *TheBB) : SPIRVInstruction(FixedWords + TheMemoryAccess.size(), OC, TheBB), SPIRVMemoryAccess(TheMemoryAccess), MemoryAccess(TheMemoryAccess), Target(TheTarget->getId()), Source(TheSource->getId()), Size(TheSize->getId()) { validate(); assert(TheBB && "Invalid BB"); updateModuleVersion(); } // Incomplete constructor SPIRVCopyMemorySized() : SPIRVInstruction(OC), SPIRVMemoryAccess(), Target(SPIRVID_INVALID), Source(SPIRVID_INVALID), Size(0) { setHasNoId(); setHasNoType(); updateModuleVersion(); } SPIRVWord getRequiredSPIRVVersion() const override { if (getSrcAlignment()) return static_cast(VersionNumber::SPIRV_1_4); return static_cast(VersionNumber::SPIRV_1_0); } SPIRVValue *getSource() { return getValue(Source); } SPIRVValue *getTarget() { return getValue(Target); } SPIRVValue *getSize() { return getValue(Size); } protected: void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); MemoryAccess.resize(TheWordCount - FixedWords); } void encode(spv_ostream &O) const override { getEncoder(O) << Target << Source << Size << MemoryAccess; } void decode(std::istream &I) override { getDecoder(I) >> Target >> Source >> Size >> MemoryAccess; memoryAccessUpdate(MemoryAccess); } void validate() const override { SPIRVInstruction::validate(); } std::vector MemoryAccess; SPIRVId Target; SPIRVId Source; SPIRVId Size; }; class SPIRVVectorExtractDynamic : public SPIRVInstruction { public: const static Op OC = OpVectorExtractDynamic; // Complete constructor SPIRVVectorExtractDynamic(SPIRVId TheId, SPIRVValue *TheVector, SPIRVValue *TheIndex, SPIRVBasicBlock *TheBB) : SPIRVInstruction(5, OC, TheVector->getType()->getVectorComponentType(), TheId, TheBB), VectorId(TheVector->getId()), IndexId(TheIndex->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVVectorExtractDynamic() : SPIRVInstruction(OC), VectorId(SPIRVID_INVALID), IndexId(SPIRVID_INVALID) {} SPIRVValue *getVector() { return getValue(VectorId); } SPIRVValue *getIndex() const { return getValue(IndexId); } std::vector getOperands() override { return {getVector(), getIndex()}; } protected: _SPIRV_DEF_ENCDEC4(Type, Id, VectorId, IndexId) void validate() const override { SPIRVInstruction::validate(); if (getValue(VectorId)->isForward()) return; assert(getValueType(VectorId)->isTypeVector() || getValueType(VectorId)->isTypeJointMatrixINTEL()); } SPIRVId VectorId; SPIRVId IndexId; }; class SPIRVVectorInsertDynamic : public SPIRVInstruction { public: const static Op OC = OpVectorInsertDynamic; // Complete constructor SPIRVVectorInsertDynamic(SPIRVId TheId, SPIRVValue *TheVector, SPIRVValue *TheComponent, SPIRVValue *TheIndex, SPIRVBasicBlock *TheBB) : SPIRVInstruction(6, OC, TheVector->getType(), TheId, TheBB), VectorId(TheVector->getId()), IndexId(TheIndex->getId()), ComponentId(TheComponent->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVVectorInsertDynamic() : SPIRVInstruction(OC), VectorId(SPIRVID_INVALID), IndexId(SPIRVID_INVALID), ComponentId(SPIRVID_INVALID) {} SPIRVValue *getVector() { return getValue(VectorId); } SPIRVValue *getComponent() { return getValue(ComponentId); } SPIRVValue *getIndex() const { return getValue(IndexId); } std::vector getOperands() override { return {getVector(), getComponent(), getIndex()}; } protected: _SPIRV_DEF_ENCDEC5(Type, Id, VectorId, ComponentId, IndexId) void validate() const override { SPIRVInstruction::validate(); if (getValue(VectorId)->isForward()) return; assert(getValueType(VectorId)->isTypeVector() || getValueType(VectorId)->isTypeJointMatrixINTEL()); } SPIRVId VectorId; SPIRVId IndexId; SPIRVId ComponentId; }; class SPIRVVectorShuffleBase : public SPIRVInstTemplateBase { public: SPIRVValue *getVector1() { return getValue(Ops[0]); } SPIRVValue *getVector2() { return getValue(Ops[1]); } const std::vector getComponents() const { return std::vector(Ops.begin() + 2, Ops.end()); } protected: void validate() const override { SPIRVInstruction::validate(); [[maybe_unused]] SPIRVId Vector1 = Ops[0]; assert(OpCode == OpVectorShuffle); assert(Type->isTypeVector()); assert(Type->getVectorComponentType() == getValueType(Vector1)->getVectorComponentType()); assert(Ops.size() - 2 == Type->getVectorComponentCount()); } }; typedef SPIRVInstTemplate SPIRVVectorShuffle; class SPIRVControlBarrier : public SPIRVInstruction { public: static const Op OC = OpControlBarrier; // Complete constructor SPIRVControlBarrier(SPIRVValue *TheScope, SPIRVValue *TheMemScope, SPIRVValue *TheMemSema, SPIRVBasicBlock *TheBB) : SPIRVInstruction(4, OC, TheBB), ExecScope(TheScope->getId()), MemScope(TheMemScope->getId()), MemSema(TheMemSema->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVControlBarrier() : SPIRVInstruction(OC), ExecScope(ScopeInvocation) { setHasNoId(); setHasNoType(); } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); } SPIRVValue *getExecScope() const { return getValue(ExecScope); } SPIRVValue *getMemScope() const { return getValue(MemScope); } SPIRVValue *getMemSemantic() const { return getValue(MemSema); } std::vector getOperands() override { std::vector Operands; Operands.push_back(ExecScope); Operands.push_back(MemScope); Operands.push_back(MemSema); return getValues(Operands); } protected: _SPIRV_DEF_ENCDEC3(ExecScope, MemScope, MemSema) void validate() const override { assert(OpCode == OC); assert(WordCount == 4); SPIRVInstruction::validate(); } SPIRVId ExecScope; SPIRVId MemScope; SPIRVId MemSema; }; template class SPIRVLifetime : public SPIRVInstruction { public: // Complete constructor SPIRVLifetime(SPIRVId TheObject, SPIRVWord TheSize, SPIRVBasicBlock *TheBB) : SPIRVInstruction(3, OC, TheBB), Object(TheObject), Size(TheSize) { validate(); assert(TheBB && "Invalid BB"); }; // Incomplete constructor SPIRVLifetime() : SPIRVInstruction(OC), Object(SPIRVID_INVALID), Size(SPIRVWORD_MAX) { setHasNoId(); setHasNoType(); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityKernel); } SPIRVValue *getObject() { return getValue(Object); }; SPIRVWord getSize() { return Size; }; protected: void validate() const override { auto ObjType = getValue(Object)->getType(); // Type must be an OpTypePointer with Storage Class Function. assert(ObjType->isTypePointer() && "Objects type must be a pointer"); assert(static_cast(ObjType)->getStorageClass() == StorageClassFunction && "Invalid storage class"); // Size must be 0 if Pointer is a pointer to a non-void type or the // Addresses capability is not being used. If Size is non-zero, it is the // number of bytes of memory whose lifetime is starting. Its type must be an // integer type scalar. It is treated as unsigned; if its type has // Signedness of 1, its sign bit cannot be set. if (!(ObjType->getPointerElementType()->isTypeVoid() || // (void *) is i8* in LLVM IR ObjType->getPointerElementType()->isTypeInt(8)) || !Module->hasCapability(CapabilityAddresses)) assert(Size == 0 && "Size must be 0"); } _SPIRV_DEF_ENCDEC2(Object, Size) SPIRVId Object; SPIRVWord Size; }; typedef SPIRVLifetime SPIRVLifetimeStart; typedef SPIRVLifetime SPIRVLifetimeStop; class SPIRVGroupAsyncCopy : public SPIRVInstruction { public: static const Op OC = OpGroupAsyncCopy; static const SPIRVWord WC = 9; // Complete constructor SPIRVGroupAsyncCopy(SPIRVValue *TheScope, SPIRVId TheId, SPIRVValue *TheDest, SPIRVValue *TheSrc, SPIRVValue *TheNumElems, SPIRVValue *TheStride, SPIRVValue *TheEvent, SPIRVBasicBlock *TheBB) : SPIRVInstruction(WC, OC, TheEvent->getType(), TheId, TheBB), ExecScope(TheScope->getId()), Destination(TheDest->getId()), Source(TheSrc->getId()), NumElements(TheNumElems->getId()), Stride(TheStride->getId()), Event(TheEvent->getId()) { validate(); assert(TheBB && "Invalid BB"); } // Incomplete constructor SPIRVGroupAsyncCopy() : SPIRVInstruction(OC), ExecScope(SPIRVID_INVALID), Destination(SPIRVID_INVALID), Source(SPIRVID_INVALID), NumElements(SPIRVID_INVALID), Stride(SPIRVID_INVALID), Event(SPIRVID_INVALID) {} SPIRVValue *getExecScope() const { return getValue(ExecScope); } SPIRVValue *getDestination() const { return getValue(Destination); } SPIRVValue *getSource() const { return getValue(Source); } SPIRVValue *getNumElements() const { return getValue(NumElements); } SPIRVValue *getStride() const { return getValue(Stride); } SPIRVValue *getEvent() const { return getValue(Event); } std::vector getOperands() override { std::vector Operands; Operands.push_back(ExecScope); Operands.push_back(Destination); Operands.push_back(Source); Operands.push_back(NumElements); Operands.push_back(Stride); Operands.push_back(Event); return getValues(Operands); } protected: _SPIRV_DEF_ENCDEC8(Type, Id, ExecScope, Destination, Source, NumElements, Stride, Event) void validate() const override { assert(OpCode == OC); assert(WordCount == WC); SPIRVInstruction::validate(); } SPIRVId ExecScope; SPIRVId Destination; SPIRVId Source; SPIRVId NumElements; SPIRVId Stride; SPIRVId Event; }; enum SPIRVOpKind { SPIRVOPK_Id, SPIRVOPK_Literal, SPIRVOPK_Count }; class SPIRVDevEnqInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityDeviceEnqueue); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // CL 2.0 enqueue kernel builtins _SPIRV_OP(EnqueueMarker, true, 7) _SPIRV_OP(EnqueueKernel, true, 13, true) _SPIRV_OP(GetKernelNDrangeSubGroupCount, true, 8) _SPIRV_OP(GetKernelNDrangeMaxSubGroupSize, true, 8) _SPIRV_OP(GetKernelWorkGroupSize, true, 7) _SPIRV_OP(GetKernelPreferredWorkGroupSizeMultiple, true, 7) _SPIRV_OP(RetainEvent, false, 2) _SPIRV_OP(ReleaseEvent, false, 2) _SPIRV_OP(CreateUserEvent, true, 3) _SPIRV_OP(IsValidEvent, true, 4) _SPIRV_OP(SetUserEventStatus, false, 3) _SPIRV_OP(CaptureEventProfilingInfo, false, 4) _SPIRV_OP(GetDefaultQueue, true, 3) _SPIRV_OP(BuildNDRange, true, 6) #undef _SPIRV_OP class SPIRVPipeInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityPipes); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // CL 2.0 pipe builtins _SPIRV_OP(ReadPipe, true, 7) _SPIRV_OP(WritePipe, true, 7) _SPIRV_OP(ReservedReadPipe, true, 9) _SPIRV_OP(ReservedWritePipe, true, 9) _SPIRV_OP(ReserveReadPipePackets, true, 7) _SPIRV_OP(ReserveWritePipePackets, true, 7) _SPIRV_OP(CommitReadPipe, false, 5) _SPIRV_OP(CommitWritePipe, false, 5) _SPIRV_OP(IsValidReserveId, true, 4) _SPIRV_OP(GetNumPipePackets, true, 6) _SPIRV_OP(GetMaxPipePackets, true, 6) #undef _SPIRV_OP class SPIRVPipeStorageInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityPipeStorage, CapabilityPipes); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(CreatePipeFromPipeStorage, true, 4) #undef _SPIRV_OP class SPIRVGroupInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroups); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // Group instructions. // Even though GroupWaitEvents has Group in its name, it doesn't require the // Group capability typedef SPIRVInstTemplate SPIRVGroupWaitEvents; _SPIRV_OP(GroupAll, true, 5) _SPIRV_OP(GroupAny, true, 5) _SPIRV_OP(GroupBroadcast, true, 6) _SPIRV_OP(GroupIAdd, true, 6, false, 1) _SPIRV_OP(GroupFAdd, true, 6, false, 1) _SPIRV_OP(GroupFMin, true, 6, false, 1) _SPIRV_OP(GroupUMin, true, 6, false, 1) _SPIRV_OP(GroupSMin, true, 6, false, 1) _SPIRV_OP(GroupFMax, true, 6, false, 1) _SPIRV_OP(GroupUMax, true, 6, false, 1) _SPIRV_OP(GroupSMax, true, 6, false, 1) _SPIRV_OP(GroupReserveReadPipePackets, true, 8) _SPIRV_OP(GroupReserveWritePipePackets, true, 8) _SPIRV_OP(GroupCommitReadPipe, false, 6) _SPIRV_OP(GroupCommitWritePipe, false, 6) #undef _SPIRV_OP class SPIRVGroupNonUniformElectInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniform); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformElect, true, 4) #undef _SPIRV_OP class SPIRVGroupNonUniformVoteInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniformVote); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformAll, true, 5) _SPIRV_OP(GroupNonUniformAny, true, 5) _SPIRV_OP(GroupNonUniformAllEqual, true, 5) #undef _SPIRV_OP class SPIRVGroupNonUniformBallotInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniformBallot); } SPIRVWord getRequiredSPIRVVersion() const override { switch (OpCode) { case OpGroupNonUniformBroadcast: { assert(Ops.size() == 3 && "Expecting (Execution, Value, Id) operands"); if (!isConstantOpCode(getOperand(2)->getOpCode())) { // Before version 1.5, Id must come from a constant instruction. return static_cast(VersionNumber::SPIRV_1_5); } break; } default: break; } return static_cast(VersionNumber::SPIRV_1_3); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformBroadcast, true, 6) _SPIRV_OP(GroupNonUniformBroadcastFirst, true, 5) _SPIRV_OP(GroupNonUniformBallot, true, 5) _SPIRV_OP(GroupNonUniformInverseBallot, true, 5) _SPIRV_OP(GroupNonUniformBallotBitExtract, true, 6) _SPIRV_OP(GroupNonUniformBallotBitCount, true, 6, false, 1) _SPIRV_OP(GroupNonUniformBallotFindLSB, true, 5) _SPIRV_OP(GroupNonUniformBallotFindMSB, true, 5) #undef _SPIRV_OP class SPIRVGroupNonUniformArithmeticInst : public SPIRVInstTemplateBase { public: void setOpWords(const std::vector &Ops) override { SPIRVInstTemplateBase::setOpWords(Ops); SPIRVGroupOperationKind GroupOp; if (getSPIRVGroupOperation(GroupOp)) { if (GroupOp == GroupOperationClusteredReduce) Module->addCapability(CapabilityGroupNonUniformClustered); else Module->addCapability(CapabilityGroupNonUniformArithmetic); } else llvm_unreachable( "GroupNonUniformArithmeticInst has no group operation operand!"); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformIAdd, true, 6, true, 1) _SPIRV_OP(GroupNonUniformFAdd, true, 6, true, 1) _SPIRV_OP(GroupNonUniformIMul, true, 6, true, 1) _SPIRV_OP(GroupNonUniformFMul, true, 6, true, 1) _SPIRV_OP(GroupNonUniformSMin, true, 6, true, 1) _SPIRV_OP(GroupNonUniformUMin, true, 6, true, 1) _SPIRV_OP(GroupNonUniformFMin, true, 6, true, 1) _SPIRV_OP(GroupNonUniformSMax, true, 6, true, 1) _SPIRV_OP(GroupNonUniformUMax, true, 6, true, 1) _SPIRV_OP(GroupNonUniformFMax, true, 6, true, 1) _SPIRV_OP(GroupNonUniformBitwiseAnd, true, 6, true, 1) _SPIRV_OP(GroupNonUniformBitwiseOr, true, 6, true, 1) _SPIRV_OP(GroupNonUniformBitwiseXor, true, 6, true, 1) _SPIRV_OP(GroupNonUniformLogicalAnd, true, 6, true, 1) _SPIRV_OP(GroupNonUniformLogicalOr, true, 6, true, 1) _SPIRV_OP(GroupNonUniformLogicalXor, true, 6, true, 1) #undef _SPIRV_OP class SPIRVGroupNonUniformShuffleInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniformShuffle); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformShuffle, true, 6) _SPIRV_OP(GroupNonUniformShuffleXor, true, 6) #undef _SPIRV_OP class SPIRVGroupNonUniformShuffleRelativeInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniformShuffleRelative); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformShuffleUp, true, 6) _SPIRV_OP(GroupNonUniformShuffleDown, true, 6) #undef _SPIRV_OP class SPIRVGroupNonUniformRotateKHRInst : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupNonUniformRotateKHR); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_subgroup_rotate; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(GroupNonUniformRotateKHR, true, 6, true) #undef _SPIRV_OP class SPIRVBlockingPipesIntelInst : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityBlockingPipesINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_blocking_pipes; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(ReadPipeBlockingINTEL, false, 5) _SPIRV_OP(WritePipeBlockingINTEL, false, 5) #undef _SPIRV_OP class SPIRVFixedPointIntelInst : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityArbitraryPrecisionFixedPointINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_arbitrary_precision_fixed_point; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(FixedSqrtINTEL, true, 9) _SPIRV_OP(FixedRecipINTEL, true, 9) _SPIRV_OP(FixedRsqrtINTEL, true, 9) _SPIRV_OP(FixedSinINTEL, true, 9) _SPIRV_OP(FixedCosINTEL, true, 9) _SPIRV_OP(FixedSinCosINTEL, true, 9) _SPIRV_OP(FixedSinPiINTEL, true, 9) _SPIRV_OP(FixedCosPiINTEL, true, 9) _SPIRV_OP(FixedSinCosPiINTEL, true, 9) _SPIRV_OP(FixedLogINTEL, true, 9) _SPIRV_OP(FixedExpINTEL, true, 9) #undef _SPIRV_OP class SPIRVArbFloatIntelInst : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityArbitraryPrecisionFloatingPointINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_arbitrary_precision_floating_point; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRVArbitraryFloat##x##INTEL; _SPIRV_OP(Cast, true, 9) _SPIRV_OP(CastFromInt, true, 9) _SPIRV_OP(CastToInt, true, 9) _SPIRV_OP(Add, true, 11) _SPIRV_OP(Sub, true, 11) _SPIRV_OP(Mul, true, 11) _SPIRV_OP(Div, true, 11) _SPIRV_OP(GT, true, 7) _SPIRV_OP(GE, true, 7) _SPIRV_OP(LT, true, 7) _SPIRV_OP(LE, true, 7) _SPIRV_OP(EQ, true, 7) _SPIRV_OP(Recip, true, 9) _SPIRV_OP(RSqrt, true, 9) _SPIRV_OP(Cbrt, true, 9) _SPIRV_OP(Hypot, true, 11) _SPIRV_OP(Sqrt, true, 9) _SPIRV_OP(Log, true, 9) _SPIRV_OP(Log2, true, 9) _SPIRV_OP(Log10, true, 9) _SPIRV_OP(Log1p, true, 9) _SPIRV_OP(Exp, true, 9) _SPIRV_OP(Exp2, true, 9) _SPIRV_OP(Exp10, true, 9) _SPIRV_OP(Expm1, true, 9) _SPIRV_OP(Sin, true, 9) _SPIRV_OP(Cos, true, 9) _SPIRV_OP(SinCos, true, 9) _SPIRV_OP(SinPi, true, 9) _SPIRV_OP(CosPi, true, 9) _SPIRV_OP(SinCosPi, true, 9) _SPIRV_OP(ASin, true, 9) _SPIRV_OP(ASinPi, true, 9) _SPIRV_OP(ACos, true, 9) _SPIRV_OP(ACosPi, true, 9) _SPIRV_OP(ATan, true, 9) _SPIRV_OP(ATanPi, true, 9) _SPIRV_OP(ATan2, true, 11) _SPIRV_OP(Pow, true, 11) _SPIRV_OP(PowR, true, 11) _SPIRV_OP(PowN, true, 11) #undef _SPIRV_OP class SPIRVAtomicInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { // Most of the atomic instructions require a specific capability when // operating on 64-bit integers. // In SPIRV 1.2 spec, only 2 atomic instructions have no result type: // 1. OpAtomicStore - need to check type of the Value operand // 2. OpAtomicFlagClear - doesn't require Int64Atomics capability. // Besides, OpAtomicCompareExchangeWeak, OpAtomicFlagTestAndSet and // OpAtomicFlagClear instructions require the "kernel" capability. But this // capability should be added by setting the OpenCL memory model. if (hasType() && getType()->isTypeInt(64)) return {CapabilityInt64Atomics}; return {}; } // Overriding the following method because of particular OpAtomic* // instructions that declare additional capabilities, e.g. based on operand // types. void setOpWords(const std::vector &TheOps) override { SPIRVInstTemplateBase::setOpWords(TheOps); for (auto RC : getRequiredCapability()) Module->addCapability(RC); } }; class SPIRVAtomicStoreInst : public SPIRVAtomicInstBase { public: // Overriding the following method because of 'const'-related // issues with overriding getRequiredCapability(). TODO: Resolve. void setOpWords(const std::vector &TheOps) override { SPIRVInstTemplateBase::setOpWords(TheOps); static const unsigned ValueOperandIndex = 3; if (getOperand(ValueOperandIndex)->getType()->isTypeInt(64)) Module->addCapability(CapabilityInt64Atomics); } }; class SPIRVAtomicFAddEXTInst : public SPIRVAtomicInstBase { public: llvm::Optional getRequiredExtension() const override { assert(hasType()); if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) return ExtensionID::SPV_INTEL_shader_atomic_bfloat16; if (getType()->isTypeFloat(16)) return ExtensionID::SPV_EXT_shader_atomic_float16_add; return ExtensionID::SPV_EXT_shader_atomic_float_add; } SPIRVCapVec getRequiredCapability() const override { assert(hasType()); if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) return {internal::CapabilityAtomicBFloat16AddINTEL}; if (getType()->isTypeFloat(16)) return {CapabilityAtomicFloat16AddEXT}; if (getType()->isTypeFloat(32)) return {CapabilityAtomicFloat32AddEXT}; if (getType()->isTypeFloat(64)) return {CapabilityAtomicFloat64AddEXT}; llvm_unreachable( "AtomicFAddEXT can only be generated for bf16, f16, f32, f64 types"); } }; class SPIRVAtomicFMinMaxEXTBase : public SPIRVAtomicInstBase { public: llvm::Optional getRequiredExtension() const override { if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) return ExtensionID::SPV_INTEL_shader_atomic_bfloat16; return ExtensionID::SPV_EXT_shader_atomic_float_min_max; } SPIRVCapVec getRequiredCapability() const override { assert(hasType()); if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) return {internal::CapabilityAtomicBFloat16MinMaxINTEL}; if (getType()->isTypeFloat(16)) return {CapabilityAtomicFloat16MinMaxEXT}; if (getType()->isTypeFloat(32)) return {CapabilityAtomicFloat32MinMaxEXT}; if (getType()->isTypeFloat(64)) return {CapabilityAtomicFloat64MinMaxEXT}; llvm_unreachable("AtomicF(Min|Max)EXT can only be generated for bf16, f16, " "f32, f64 types"); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // Atomic builtins _SPIRV_OP(AtomicFlagTestAndSet, true, 6) _SPIRV_OP(AtomicFlagClear, false, 4) _SPIRV_OP(AtomicLoad, true, 6) _SPIRV_OP(AtomicExchange, true, 7) _SPIRV_OP(AtomicCompareExchange, true, 9) _SPIRV_OP(AtomicCompareExchangeWeak, true, 9) _SPIRV_OP(AtomicIIncrement, true, 6) _SPIRV_OP(AtomicIDecrement, true, 6) _SPIRV_OP(AtomicIAdd, true, 7) _SPIRV_OP(AtomicISub, true, 7) _SPIRV_OP(AtomicUMin, true, 7) _SPIRV_OP(AtomicUMax, true, 7) _SPIRV_OP(AtomicSMin, true, 7) _SPIRV_OP(AtomicSMax, true, 7) _SPIRV_OP(AtomicAnd, true, 7) _SPIRV_OP(AtomicOr, true, 7) _SPIRV_OP(AtomicXor, true, 7) _SPIRV_OP(MemoryBarrier, false, 3) #undef _SPIRV_OP #define _SPIRV_OP(x, BaseClass, ...) \ typedef SPIRVInstTemplate SPIRV##x; // Specialized atomic builtins _SPIRV_OP(AtomicStore, AtomicStoreInst, false, 5) _SPIRV_OP(AtomicFAddEXT, AtomicFAddEXTInst, true, 7) _SPIRV_OP(AtomicFMinEXT, AtomicFMinMaxEXTBase, true, 7) _SPIRV_OP(AtomicFMaxEXT, AtomicFMinMaxEXTBase, true, 7) #undef _SPIRV_OP class SPIRVImageInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityImageBasic); } protected: void setOpWords(const std::vector &OpsArg) override; }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // Image instructions _SPIRV_OP(SampledImage, true, 5) _SPIRV_OP(ImageSampleImplicitLod, true, 5, true) _SPIRV_OP(ImageSampleExplicitLod, true, 7, true, 2) _SPIRV_OP(ImageRead, true, 5, true, 2) _SPIRV_OP(ImageWrite, false, 4, true, 3) _SPIRV_OP(ImageQueryFormat, true, 4) _SPIRV_OP(ImageQueryOrder, true, 4) _SPIRV_OP(ImageQuerySizeLod, true, 5) _SPIRV_OP(ImageQuerySize, true, 4) _SPIRV_OP(ImageQueryLod, true, 5) _SPIRV_OP(ImageQueryLevels, true, 4) _SPIRV_OP(ImageQuerySamples, true, 4) #undef _SPIRV_OP #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; // Other instructions _SPIRV_OP(GenericPtrMemSemantics, true, 4, false) _SPIRV_OP(GenericCastToPtrExplicit, true, 5, false, 1) #undef _SPIRV_OP class SPIRVSpecConstantOpBase : public SPIRVInstTemplateBase { public: bool isOperandLiteral(unsigned I) const override { // If SpecConstant results from CompositeExtract/Insert operation, then all // operands are expected to be literals. switch (Ops[0]) { // Opcode of underlying SpecConstant operation case OpCompositeExtract: case OpCompositeInsert: return true; default: return SPIRVInstTemplateBase::isOperandLiteral(I); } } }; typedef SPIRVInstTemplate SPIRVSpecConstantOp; class SPIRVAssumeTrueKHR : public SPIRVInstruction { public: static const Op OC = OpAssumeTrueKHR; static const SPIRVWord FixedWordCount = 2; SPIRVAssumeTrueKHR(SPIRVId TheCondition, SPIRVBasicBlock *BB) : SPIRVInstruction(FixedWordCount, OC, BB), ConditionId(TheCondition) { validate(); setHasNoId(); setHasNoType(); assert(BB && "Invalid BB"); } SPIRVAssumeTrueKHR() : SPIRVInstruction(OC), ConditionId(SPIRVID_MAX) { setHasNoId(); setHasNoType(); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityExpectAssumeKHR); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_expect_assume; } SPIRVValue *getCondition() const { return getValue(ConditionId); } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); } _SPIRV_DEF_ENCDEC1(ConditionId) protected: SPIRVId ConditionId; }; class SPIRVExpectKHRInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityExpectAssumeKHR); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_expect_assume; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(ExpectKHR, true, 5) #undef _SPIRV_OP class SPIRVDotKHRBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { // Both vector operands must have the same type, so analyzing the // first operand will suffice. SPIRVCapabilityKind ArgCap = getRequiredCapabilityForOperand(Ops[0]); return getVec(ArgCap, CapabilityDotProductKHR); } llvm::Optional getRequiredExtension() const override { if (!Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) return ExtensionID::SPV_KHR_integer_dot_product; return {}; } void validate() const override { SPIRVInstruction::validate(); SPIRVId Vec1 = Ops[0]; SPIRVId Vec2 = Ops[1]; (void)Vec1; (void)Vec2; assert(getValueType(Vec1) == getValueType(Vec2) && "Input vectors must have the same type"); assert(getType()->isTypeInt() && "Result type must be an integer type"); assert(!getType()->isTypeVector() && "Result type must be scalar"); } private: bool isAccSat() const { return (OpCode == OpSDotAccSatKHR || OpCode == OpUDotAccSatKHR || OpCode == OpSUDotAccSatKHR); } Optional getPackedVectorFormat() const { size_t PackFmtIdx = 2; if (isAccSat()) { // AccSat instructions have an additional Accumulator operand. PackFmtIdx++; } if (PackFmtIdx == Ops.size() - 1) return static_cast(Ops[PackFmtIdx]); return None; } SPIRVCapabilityKind getRequiredCapabilityForOperand(SPIRVId ArgId) const { const SPIRVType *T = getValueType(ArgId); if (auto PackFmt = getPackedVectorFormat()) { switch (*PackFmt) { case PackedVectorFormatPackedVectorFormat4x8BitKHR: assert(!T->isTypeVector() && T->isTypeInt() && T->getBitWidth() == 32 && "Type does not match pack format"); return CapabilityDotProductInput4x8BitPackedKHR; case PackedVectorFormatMax: break; } llvm_unreachable("Unknown Packed Vector Format"); } if (T->isTypeVector()) { const SPIRVType *EltT = T->getVectorComponentType(); if (T->getVectorComponentCount() == 4 && EltT->isTypeInt() && EltT->getBitWidth() == 8) return CapabilityDotProductInput4x8BitKHR; if (EltT->isTypeInt()) return CapabilityDotProductInputAllKHR; } llvm_unreachable("No mapping for argument type to capability."); } SPIRVWord getRequiredSPIRVVersion() const override { if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_6)) return static_cast(VersionNumber::SPIRV_1_6); return static_cast(VersionNumber::SPIRV_1_0); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; _SPIRV_OP(SDotKHR, true, 5, true, 2) _SPIRV_OP(UDotKHR, true, 5, true, 2) _SPIRV_OP(SUDotKHR, true, 5, true, 2) _SPIRV_OP(SDotAccSatKHR, true, 6, true, 3) _SPIRV_OP(UDotAccSatKHR, true, 6, true, 3) _SPIRV_OP(SUDotAccSatKHR, true, 6, true, 3) #undef _SPIRV_OP class SPIRVBitOp : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { if (Module->isAllowedToUseExtension(ExtensionID::SPV_KHR_bit_instructions)) return getVec(CapabilityBitInstructions); return getVec(CapabilityShader); } llvm::Optional getRequiredExtension() const override { for (auto Cap : getRequiredCapability()) { if (Cap == CapabilityBitInstructions) return ExtensionID::SPV_KHR_bit_instructions; } return None; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate SPIRV##x; _SPIRV_OP(BitFieldInsert, true, 7) _SPIRV_OP(BitFieldSExtract, true, 6) _SPIRV_OP(BitFieldUExtract, true, 6) _SPIRV_OP(BitReverse, true, 4) #undef _SPIRV_OP class SPIRVSubgroupShuffleINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupShuffleINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_subgroups; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; // Intel Subgroup Shuffle Instructions _SPIRV_OP(SubgroupShuffleINTEL, true, 5) _SPIRV_OP(SubgroupShuffleDownINTEL, true, 6) _SPIRV_OP(SubgroupShuffleUpINTEL, true, 6) _SPIRV_OP(SubgroupShuffleXorINTEL, true, 5) #undef _SPIRV_OP class SPIRVSubgroupBufferBlockIOINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupBufferBlockIOINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_subgroups; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; // Intel Subgroup Buffer Block Read and Write Instructions _SPIRV_OP(SubgroupBlockReadINTEL, true, 4) _SPIRV_OP(SubgroupBlockWriteINTEL, false, 3) #undef _SPIRV_OP class SPIRVSubgroupImageBlockIOINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupImageBlockIOINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_subgroups; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; // Intel Subgroup Image Block Read and Write Instructions _SPIRV_OP(SubgroupImageBlockReadINTEL, true, 5) _SPIRV_OP(SubgroupImageBlockWriteINTEL, false, 4) #undef _SPIRV_OP class SPIRVSubgroupImageMediaBlockIOINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupImageMediaBlockIOINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_media_block_io; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; // Intel Subgroup Image Media Block Read and Write Instructions _SPIRV_OP(SubgroupImageMediaBlockReadINTEL, true, 7) _SPIRV_OP(SubgroupImageMediaBlockWriteINTEL, false, 6) #undef _SPIRV_OP class SPIRVSubgroupAVCIntelInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupAvcMotionEstimationINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_device_side_avc_motion_estimation; } }; // Intel Subgroup AVC Motion Estimation Instructions typedef SPIRVInstTemplate SPIRVVmeImageINTEL; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRVSubgroupAvc##x##INTEL; _SPIRV_OP(MceGetDefaultInterBaseMultiReferencePenalty, true, 5) _SPIRV_OP(MceSetInterBaseMultiReferencePenalty, true, 5) _SPIRV_OP(MceGetDefaultInterShapePenalty, true, 5) _SPIRV_OP(MceSetInterShapePenalty, true, 5) _SPIRV_OP(MceGetDefaultInterDirectionPenalty, true, 5) _SPIRV_OP(MceSetInterDirectionPenalty, true, 5) _SPIRV_OP(MceGetDefaultInterMotionVectorCostTable, true, 5) _SPIRV_OP(MceGetDefaultHighPenaltyCostTable, true, 3) _SPIRV_OP(MceGetDefaultMediumPenaltyCostTable, true, 3) _SPIRV_OP(MceGetDefaultLowPenaltyCostTable, true, 3) _SPIRV_OP(MceSetMotionVectorCostFunction, true, 7) _SPIRV_OP(MceSetAcOnlyHaar, true, 4) _SPIRV_OP(MceSetSourceInterlacedFieldPolarity, true, 5) _SPIRV_OP(MceSetSingleReferenceInterlacedFieldPolarity, true, 5) _SPIRV_OP(MceSetDualReferenceInterlacedFieldPolarities, true, 6) _SPIRV_OP(MceConvertToImePayload, true, 4) _SPIRV_OP(MceConvertToImeResult, true, 4) _SPIRV_OP(MceConvertToRefPayload, true, 4) _SPIRV_OP(MceConvertToRefResult, true, 4) _SPIRV_OP(MceConvertToSicPayload, true, 4) _SPIRV_OP(MceConvertToSicResult, true, 4) _SPIRV_OP(MceGetMotionVectors, true, 4) _SPIRV_OP(MceGetInterDistortions, true, 4) _SPIRV_OP(MceGetBestInterDistortions, true, 4) _SPIRV_OP(MceGetInterMajorShape, true, 4) _SPIRV_OP(MceGetInterMinorShape, true, 4) _SPIRV_OP(MceGetInterDirections, true, 4) _SPIRV_OP(MceGetInterMotionVectorCount, true, 4) _SPIRV_OP(MceGetInterReferenceIds, true, 4) _SPIRV_OP(MceGetInterReferenceInterlacedFieldPolarities, true, 6) _SPIRV_OP(ImeInitialize, true, 6) _SPIRV_OP(ImeSetSingleReference, true, 6) _SPIRV_OP(ImeSetDualReference, true, 7) _SPIRV_OP(ImeRefWindowSize, true, 5) _SPIRV_OP(ImeAdjustRefOffset, true, 7) _SPIRV_OP(ImeConvertToMcePayload, true, 4) _SPIRV_OP(ImeSetMaxMotionVectorCount, true, 5) _SPIRV_OP(ImeSetUnidirectionalMixDisable, true, 4) _SPIRV_OP(ImeSetEarlySearchTerminationThreshold, true, 5) _SPIRV_OP(ImeSetWeightedSad, true, 5) _SPIRV_OP(ImeEvaluateWithSingleReference, true, 6) _SPIRV_OP(ImeEvaluateWithDualReference, true, 7) _SPIRV_OP(ImeEvaluateWithSingleReferenceStreamin, true, 7) _SPIRV_OP(ImeEvaluateWithDualReferenceStreamin, true, 8) _SPIRV_OP(ImeEvaluateWithSingleReferenceStreamout, true, 6) _SPIRV_OP(ImeEvaluateWithDualReferenceStreamout, true, 7) _SPIRV_OP(ImeEvaluateWithSingleReferenceStreaminout, true, 7) _SPIRV_OP(ImeEvaluateWithDualReferenceStreaminout, true, 8) _SPIRV_OP(ImeConvertToMceResult, true, 4) _SPIRV_OP(ImeGetSingleReferenceStreamin, true, 4) _SPIRV_OP(ImeGetDualReferenceStreamin, true, 4) _SPIRV_OP(ImeStripSingleReferenceStreamout, true, 4) _SPIRV_OP(ImeStripDualReferenceStreamout, true, 4) _SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeMotionVectors, true, 5) _SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeDistortions, true, 5) _SPIRV_OP(ImeGetStreamoutSingleReferenceMajorShapeReferenceIds, true, 5) _SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeMotionVectors, true, 6) _SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeDistortions, true, 6) _SPIRV_OP(ImeGetStreamoutDualReferenceMajorShapeReferenceIds, true, 6) _SPIRV_OP(ImeGetBorderReached, true, 5) _SPIRV_OP(ImeGetTruncatedSearchIndication, true, 4) _SPIRV_OP(ImeGetUnidirectionalEarlySearchTermination, true, 4) _SPIRV_OP(ImeGetWeightingPatternMinimumMotionVector, true, 4) _SPIRV_OP(ImeGetWeightingPatternMinimumDistortion, true, 4) _SPIRV_OP(FmeInitialize, true, 10) _SPIRV_OP(BmeInitialize, true, 11) _SPIRV_OP(RefConvertToMcePayload, true, 4) _SPIRV_OP(RefSetBidirectionalMixDisable, true, 4) _SPIRV_OP(RefSetBilinearFilterEnable, true, 4) _SPIRV_OP(RefEvaluateWithSingleReference, true, 6) _SPIRV_OP(RefEvaluateWithDualReference, true, 7) _SPIRV_OP(RefEvaluateWithMultiReference, true, 6) _SPIRV_OP(RefEvaluateWithMultiReferenceInterlaced, true, 7) _SPIRV_OP(RefConvertToMceResult, true, 4) _SPIRV_OP(SicInitialize, true, 4) _SPIRV_OP(SicConfigureSkc, true, 9) _SPIRV_OP(SicGetMotionVectorMask, true, 5) _SPIRV_OP(SicConvertToMcePayload, true, 4) _SPIRV_OP(SicSetIntraLumaShapePenalty, true, 5) _SPIRV_OP(SicSetBilinearFilterEnable, true, 4) _SPIRV_OP(SicSetSkcForwardTransformEnable, true, 5) _SPIRV_OP(SicSetBlockBasedRawSkipSad, true, 5) _SPIRV_OP(SicEvaluateWithSingleReference, true, 6) _SPIRV_OP(SicEvaluateWithDualReference, true, 7) _SPIRV_OP(SicEvaluateWithMultiReference, true, 6) _SPIRV_OP(SicEvaluateWithMultiReferenceInterlaced, true, 7) _SPIRV_OP(SicConvertToMceResult, true, 4) _SPIRV_OP(SicGetInterRawSads, true, 4) #undef _SPIRV_OP class SPIRVSubgroupAVCIntelInstBaseIntra : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupAvcMotionEstimationIntraINTEL); } }; // Intel Subgroup AVC Motion Estimation Intra Instructions #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRVSubgroupAvc##x##INTEL; _SPIRV_OP(MceGetDefaultIntraLumaShapePenalty, true, 5) _SPIRV_OP(MceGetDefaultIntraLumaModePenalty, true, 5) _SPIRV_OP(MceGetDefaultNonDcLumaIntraPenalty, true, 3) _SPIRV_OP(SicConfigureIpeLuma, true, 11) _SPIRV_OP(SicSetIntraLumaModeCostFunction, true, 7) _SPIRV_OP(SicEvaluateIpe, true, 5) _SPIRV_OP(SicGetIpeLumaShape, true, 4) _SPIRV_OP(SicGetBestIpeLumaDistortion, true, 4) _SPIRV_OP(SicGetPackedIpeLumaModes, true, 4) _SPIRV_OP(SicGetPackedSkcLumaCountThreshold, true, 4) _SPIRV_OP(SicGetPackedSkcLumaSumThreshold, true, 4) #undef _SPIRV_OP class SPIRVSubgroupAVCIntelInstBaseChroma : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupAvcMotionEstimationChromaINTEL); } }; // Intel Subgroup AVC Motion Estimation Chroma Instructions #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRVSubgroupAvc##x##INTEL; _SPIRV_OP(MceGetDefaultIntraChromaModeBasePenalty, true, 3) _SPIRV_OP(SicConfigureIpeLumaChroma, true, 14) _SPIRV_OP(SicSetIntraChromaModeCostFunction, true, 5) _SPIRV_OP(SicGetBestIpeChromaDistortion, true, 4) _SPIRV_OP(SicGetIpeChromaMode, true, 4) #undef _SPIRV_OP SPIRVSpecConstantOp *createSpecConstantOpInst(SPIRVInstruction *Inst); SPIRVInstruction *createInstFromSpecConstantOp(SPIRVSpecConstantOp *C); class SPIRVVariableLengthArrayINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityVariableLengthArrayINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_variable_length_array; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(VariableLengthArray, true, 4) _SPIRV_OP(SaveMemory, true, 3) _SPIRV_OP(RestoreMemory, false, 2) #undef _SPIRV_OP template class SPIRVBfloat16ConversionINTELInstBase : public SPIRVUnaryInst { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityBfloat16ConversionINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_bfloat16_conversion; } void validate() const override { SPIRVUnaryInst::validate(); SPIRVType *ResCompTy = this->getType(); SPIRVWord ResCompCount = 1; if (ResCompTy->isTypeVector()) { ResCompCount = ResCompTy->getVectorComponentCount(); ResCompTy = ResCompTy->getVectorComponentType(); } // validate is a const method, whilst getOperand is non-const method // because it may call a method of class Module that may modify LiteralMap // of Module field. That modification is not impacting validate method for // these instructions, so const_cast is safe here. using SPVBf16ConvTy = SPIRVBfloat16ConversionINTELInstBase; SPIRVValue *Input = const_cast(this)->getOperand(0); SPIRVType *InCompTy = Input->getType(); SPIRVWord InCompCount = 1; if (InCompTy->isTypeVector()) { InCompCount = InCompTy->getVectorComponentCount(); InCompTy = InCompTy->getVectorComponentType(); } auto InstName = OpCodeNameMap::map(OC); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); if (OC == internal::OpConvertFToBF16INTEL) { SPVErrLog.checkError( ResCompTy->isTypeInt(16), SPIRVEC_InvalidInstruction, InstName + "\nResult value must be a scalar or vector of integer " "16-bit type\n"); SPVErrLog.checkError( InCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction, InstName + "\nInput value must be a scalar or vector of " "floating-point 32-bit type\n"); } else { SPVErrLog.checkError( ResCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction, InstName + "\nResult value must be a scalar or vector of " "floating-point 32-bit type\n"); SPVErrLog.checkError( InCompTy->isTypeInt(16), SPIRVEC_InvalidInstruction, InstName + "\nInput value must be a scalar or vector of integer " "16-bit type\n"); } SPVErrLog.checkError( ResCompCount == InCompCount, SPIRVEC_InvalidInstruction, InstName + "\nInput type must have the same number of components as " "result type\n"); } }; #define _SPIRV_OP(x) \ typedef SPIRVBfloat16ConversionINTELInstBase SPIRV##x; _SPIRV_OP(ConvertFToBF16INTEL) _SPIRV_OP(ConvertBF16ToFINTEL) #undef _SPIRV_OP class SPIRVJointMatrixINTELInstBase : public SPIRVInstTemplateBase { protected: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_joint_matrix; } }; class SPIRVJointMatrixINTELInst : public SPIRVJointMatrixINTELInstBase { SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityJointMatrixINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(JointMatrixLoad, true, 6, true) _SPIRV_OP(JointMatrixStore, false, 5, true) _SPIRV_OP(JointMatrixMad, true, 7) _SPIRV_OP(JointMatrixSUMad, true, 7) _SPIRV_OP(JointMatrixUSMad, true, 7) _SPIRV_OP(JointMatrixUUMad, true, 7) // TODO: move to SPIRVJointMatrixINTELWorkItemInst _SPIRV_OP(JointMatrixWorkItemLength, true, 4) #undef _SPIRV_OP class SPIRVJointMatrixINTELWorkItemInst : public SPIRVJointMatrixINTELInstBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityJointMatrixWIInstructionsINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(JointMatrixGetElementCoord, true, 5) #undef _SPIRV_OP class SPIRVCooperativeMatrixPrefetchINTELInstBase : public SPIRVInstTemplateBase { protected: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_joint_matrix; } SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityCooperativeMatrixPrefetchINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(CooperativeMatrixPrefetch, false, 6, true, 3) #undef _SPIRV_OP class SPIRVCooperativeMatrixCheckedInstructionsINTELInstBase : public SPIRVInstTemplateBase { protected: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_joint_matrix; } SPIRVCapVec getRequiredCapability() const override { return getVec( internal::CapabilityCooperativeMatrixCheckedInstructionsINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate< \ SPIRVCooperativeMatrixCheckedInstructionsINTELInstBase, \ internal::Op##x##INTEL, __VA_ARGS__> \ SPIRV##x##INTEL; _SPIRV_OP(CooperativeMatrixLoadChecked, true, 9, true, 7) _SPIRV_OP(CooperativeMatrixStoreChecked, false, 8, true, 8) _SPIRV_OP(CooperativeMatrixConstructChecked, true, 8) #undef _SPIRV_OP class SPIRVCooperativeMatrixKHRInstBase : public SPIRVInstTemplateBase { protected: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_cooperative_matrix; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityCooperativeMatrixKHR); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(CooperativeMatrixLoadKHR, true, 5, true, 3) _SPIRV_OP(CooperativeMatrixStoreKHR, false, 4, true, 4) _SPIRV_OP(CooperativeMatrixLengthKHR, true, 4, false) _SPIRV_OP(CooperativeMatrixMulAddKHR, true, 6, true, 3) #undef _SPIRV_OP class SPIRVSplitBarrierINTELBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySplitBarrierINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_split_barrier; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x; _SPIRV_OP(ControlBarrierArriveINTEL, false, 4) _SPIRV_OP(ControlBarrierWaitINTEL, false, 4) #undef _SPIRV_OP class SPIRVGroupUniformArithmeticKHRInstBase : public SPIRVInstTemplateBase { public: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityGroupUniformArithmeticKHR); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_uniform_group_instructions; } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##KHR; _SPIRV_OP(GroupIMul, true, 6, false, 1) _SPIRV_OP(GroupFMul, true, 6, false, 1) _SPIRV_OP(GroupBitwiseAnd, true, 6, false, 1) _SPIRV_OP(GroupBitwiseOr, true, 6, false, 1) _SPIRV_OP(GroupBitwiseXor, true, 6, false, 1) _SPIRV_OP(GroupLogicalAnd, true, 6, false, 1) _SPIRV_OP(GroupLogicalOr, true, 6, false, 1) _SPIRV_OP(GroupLogicalXor, true, 6, false, 1) #undef _SPIRV_OP class SPIRVComplexFloat : public SPIRVInstTemplateBase { protected: void validate() const override { SPIRVId Op1 = Ops[0]; SPIRVId Op2 = Ops[1]; SPIRVType *Op1Ty, *Op2Ty; SPIRVInstruction::validate(); if (getValue(Op1)->isForward() || getValue(Op2)->isForward()) return; if (getValueType(Op1)->isTypeVector()) { Op1Ty = getValueType(Op1)->getVectorComponentType(); Op2Ty = getValueType(Op2)->getVectorComponentType(); assert(getValueType(Op1)->getVectorComponentCount() == getValueType(Op2)->getVectorComponentCount() && "Inconsistent Vector component width"); } else { Op1Ty = getValueType(Op1); Op2Ty = getValueType(Op2); } (void)Op1Ty; (void)Op2Ty; assert(Op1Ty->isTypeFloat() && "Invalid type for complex instruction"); assert(Op1Ty == Op2Ty && "Invalid type for complex instruction"); } public: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityComplexFloatMulDivINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_complex_float_mul_div; } }; template class SPIRVComplexFloatInst : public SPIRVInstTemplate {}; #define _SPIRV_OP(x) typedef SPIRVComplexFloatInst SPIRV##x; _SPIRV_OP(ComplexFMulINTEL) _SPIRV_OP(ComplexFDivINTEL) #undef _SPIRV_OP class SPIRVMaskedGatherScatterINTELInstBase : public SPIRVInstTemplateBase { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityMaskedGatherScatterINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_masked_gather_scatter; } }; class SPIRVMaskedGatherINTELInst : public SPIRVMaskedGatherScatterINTELInstBase { void validate() const override { SPIRVInstruction::validate(); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); std::string InstName = "MaskedGatherINTEL"; SPIRVType *ResTy = this->getType(); SPVErrLog.checkError(ResTy->isTypeVector(), SPIRVEC_InvalidInstruction, InstName + "\nResult must be a vector type\n"); SPIRVWord ResCompCount = ResTy->getVectorComponentCount(); SPIRVType *ResCompTy = ResTy->getVectorComponentType(); SPIRVValue *PtrVec = const_cast(this)->getOperand(0); SPIRVType *PtrVecTy = PtrVec->getType(); SPVErrLog.checkError( PtrVecTy->isTypeVectorPointer(), SPIRVEC_InvalidInstruction, InstName + "\nPtrVector must be a vector of pointers type\n"); SPIRVWord PtrVecCompCount = PtrVecTy->getVectorComponentCount(); SPIRVType *PtrVecCompTy = PtrVecTy->getVectorComponentType(); SPIRVType *PtrElemTy = PtrVecCompTy->getPointerElementType(); SPVErrLog.checkError( this->isOperandLiteral(1), SPIRVEC_InvalidInstruction, InstName + "\nAlignment must be a constant expression integer\n"); const uint32_t Align = static_cast( const_cast(this)->getOperand(2)) ->getZExtIntValue(); SPVErrLog.checkError( ((Align & (Align - 1)) == 0), SPIRVEC_InvalidInstruction, InstName + "\nAlignment must be 0 or power-of-two integer\n"); SPIRVValue *Mask = const_cast(this)->getOperand(2); SPIRVType *MaskTy = Mask->getType(); SPVErrLog.checkError(MaskTy->isTypeVector(), SPIRVEC_InvalidInstruction, InstName + "\nMask must be a vector type\n"); SPIRVType *MaskCompTy = MaskTy->getVectorComponentType(); SPVErrLog.checkError(MaskCompTy->isTypeBool(), SPIRVEC_InvalidInstruction, InstName + "\nMask must be a boolean vector type\n"); SPIRVWord MaskCompCount = MaskTy->getVectorComponentCount(); SPIRVValue *FillEmpty = const_cast(this)->getOperand(3); SPIRVType *FillEmptyTy = FillEmpty->getType(); SPVErrLog.checkError(FillEmptyTy->isTypeVector(), SPIRVEC_InvalidInstruction, InstName + "\nFillEmpty must be a vector type\n"); SPIRVWord FillEmptyCompCount = FillEmptyTy->getVectorComponentCount(); SPIRVType *FillEmptyCompTy = FillEmptyTy->getVectorComponentType(); SPVErrLog.checkError( ResCompCount == PtrVecCompCount && PtrVecCompCount == FillEmptyCompCount && FillEmptyCompCount == MaskCompCount, SPIRVEC_InvalidInstruction, InstName + "\nResult, PtrVector, Mask and FillEmpty vectors must have " "the same size\n"); SPVErrLog.checkError( ResCompTy == PtrElemTy && PtrElemTy == FillEmptyCompTy, SPIRVEC_InvalidInstruction, InstName + "\nComponent Type of Result and FillEmpty vector must be " "same as base type of PtrVector the same base type\n"); } }; class SPIRVMaskedScatterINTELInst : public SPIRVMaskedGatherScatterINTELInstBase { void validate() const override { SPIRVInstruction::validate(); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); std::string InstName = "MaskedScatterINTEL"; SPIRVValue *InputVec = const_cast(this)->getOperand(0); SPIRVType *InputVecTy = InputVec->getType(); SPVErrLog.checkError( InputVecTy->isTypeVector(), SPIRVEC_InvalidInstruction, InstName + "\nInputVector must be a vector of pointers type\n"); SPIRVWord InputVecCompCount = InputVecTy->getVectorComponentCount(); SPIRVType *InputVecCompTy = InputVecTy->getVectorComponentType(); SPIRVValue *PtrVec = const_cast(this)->getOperand(1); SPIRVType *PtrVecTy = PtrVec->getType(); SPVErrLog.checkError( PtrVecTy->isTypeVectorPointer(), SPIRVEC_InvalidInstruction, InstName + "\nPtrVector must be a vector of pointers type\n"); SPIRVWord PtrVecCompCount = PtrVecTy->getVectorComponentCount(); SPIRVType *PtrVecCompTy = PtrVecTy->getVectorComponentType(); SPIRVType *PtrElemTy = PtrVecCompTy->getPointerElementType(); SPVErrLog.checkError( this->isOperandLiteral(2), SPIRVEC_InvalidInstruction, InstName + "\nAlignment must be a constant expression integer\n"); const uint32_t Align = static_cast( const_cast(this)->getOperand(2)) ->getZExtIntValue(); SPVErrLog.checkError( ((Align & (Align - 1)) == 0), SPIRVEC_InvalidInstruction, InstName + "\nAlignment must be 0 or power-of-two integer\n"); SPIRVValue *Mask = const_cast(this)->getOperand(2); SPIRVType *MaskTy = Mask->getType(); SPVErrLog.checkError(MaskTy->isTypeVector(), SPIRVEC_InvalidInstruction, InstName + "\nMask must be a vector type\n"); SPIRVType *MaskCompTy = MaskTy->getVectorComponentType(); SPVErrLog.checkError(MaskCompTy->isTypeBool(), SPIRVEC_InvalidInstruction, InstName + "\nMask must be a boolean vector type\n"); SPIRVWord MaskCompCount = MaskTy->getVectorComponentCount(); SPVErrLog.checkError( InputVecCompCount == PtrVecCompCount && PtrVecCompCount == MaskCompCount, SPIRVEC_InvalidInstruction, InstName + "\nInputVector, PtrVector and Mask vectors must have " "the same size\n"); SPVErrLog.checkError( InputVecCompTy == PtrElemTy, SPIRVEC_InvalidInstruction, InstName + "\nComponent Type of InputVector must be " "same as base type of PtrVector the same base type\n"); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(MaskedGather, true, 7) _SPIRV_OP(MaskedScatter, false, 5) #undef _SPIRV_OP template class SPIRVTensorFloat32RoundingINTELInstBase : public SPIRVUnaryInst { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityTensorFloat32RoundingINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_tensor_float32_conversion; } void validate() const override { SPIRVUnaryInst::validate(); SPIRVType *ResCompTy = this->getType(); SPIRVWord ResCompCount = 1; if (ResCompTy->isTypeVector()) { ResCompCount = ResCompTy->getVectorComponentCount(); ResCompTy = ResCompTy->getVectorComponentType(); } // validate is a const method, whilst getOperand is non-const method // because it may call a method of class Module that may modify LiteralMap // of Module field. That modification is not impacting validate method for // these instructions, so const_cast is safe here. using SPVTF32RoundTy = SPIRVTensorFloat32RoundingINTELInstBase; SPIRVValue *Input = const_cast(this)->getOperand(0); SPIRVType *InCompTy = Input->getType(); SPIRVWord InCompCount = 1; if (InCompTy->isTypeVector()) { InCompCount = InCompTy->getVectorComponentCount(); InCompTy = InCompTy->getVectorComponentType(); } auto InstName = OpCodeNameMap::map(OC); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); SPVErrLog.checkError( ResCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction, InstName + "\nResult value must be a scalar or vector of floating-point" " 32-bit type\n"); SPVErrLog.checkError(InCompTy->isTypeFloat(32), SPIRVEC_InvalidInstruction, InstName + "\nInput value must be a scalar or vector of " "floating-point 32-bit type\n"); SPVErrLog.checkError( ResCompCount == InCompCount, SPIRVEC_InvalidInstruction, InstName + "\nInput type must have the same number of components as " "result type\n"); } }; #define _SPIRV_OP(x) \ typedef SPIRVTensorFloat32RoundingINTELInstBase SPIRV##x; _SPIRV_OP(RoundFToTF32INTEL) #undef _SPIRV_OP template class SPIRVReadClockKHRInstBase : public SPIRVUnaryInst { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityShaderClockKHR); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_KHR_shader_clock; } void validate() const override { SPIRVUnaryInst::validate(); SPIRVType *ResCompTy = this->getType(); SPIRVWord ResCompCount = 1; if (ResCompTy->isTypeVector()) { ResCompCount = ResCompTy->getVectorComponentCount(); ResCompTy = ResCompTy->getVectorComponentType(); } auto InstName = OpCodeNameMap::map(OC); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); // check for either 64 bit int type or two element vector of 32 bit int // types. SPVErrLog.checkError( ResCompTy->isTypeInt(64) || (ResCompCount == 2 && ResCompTy->isTypeInt(32)), SPIRVEC_InvalidInstruction, InstName + "\nResult value must be a scalar of integer" " 64-bit type or two element vector of 32-bit type\n"); } }; #define _SPIRV_OP(x, ...) typedef SPIRVReadClockKHRInstBase SPIRV##x; _SPIRV_OP(ReadClockKHR) #undef _SPIRV_OP template class SPIRVBindlessImagesInstBase : public SPIRVUnaryInst { protected: SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityBindlessImagesINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_bindless_images; } void validate() const override { SPIRVUnary::validate(); // validate is a const method, whilst getOperand is non-const method // because it may call a method of class Module that may modify LiteralMap // of Module field. That modification is not impacting validate method for // these instructions, so const_cast is safe here. using SPVBindlessImagesInst = SPIRVBindlessImagesInstBase; SPIRVValue *Input = const_cast(this)->getOperand(0); SPIRVType *InCompTy = Input->getType(); auto StringAddrMod = [](SPIRVAddressingModelKind Kind) -> std::string { if (Kind == AddressingModelPhysical32) return std::string("Physical32"); if (Kind == AddressingModelPhysical64) return std::string("Physical64"); return std::string("AddressingModel: ") + std::to_string(Kind); }; auto InstName = OpCodeNameMap::map(OC); auto AddrMod = this->getModule()->getAddressingModel(); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); SPVErrLog.checkError( (InCompTy->isTypeInt(32) && AddrMod == AddressingModelPhysical32) || (InCompTy->isTypeInt(64) && AddrMod == AddressingModelPhysical64), SPIRVEC_InvalidInstruction, InstName + "\nParameter value must be a 32-bit scalar in case of " "Physical32 addressing model or a 64-bit scalar in case of " "Physical64 addressing model\n" "Type size: " + std::to_string(InCompTy->getBitWidth()) + "\nAddressing model: " + StringAddrMod(AddrMod) + "\n"); SPIRVType *ResTy = this->getType(); SPVErrLog.checkError( (ResTy->isTypeImage() && OC == internal::OpConvertHandleToImageINTEL) || (ResTy->isTypeSampler() && OC == internal::OpConvertHandleToSamplerINTEL) || (ResTy->isTypeSampledImage() && OC == internal::OpConvertHandleToSampledImageINTEL), SPIRVEC_InvalidInstruction, InstName + "\nIncorrect return type of the instruction must be " "image/sampler\n"); } }; #define _SPIRV_OP(x) \ typedef SPIRVBindlessImagesInstBase SPIRV##x; _SPIRV_OP(ConvertHandleToImageINTEL) _SPIRV_OP(ConvertHandleToSamplerINTEL) _SPIRV_OP(ConvertHandleToSampledImageINTEL) #undef _SPIRV_OP class SPIRVSubgroup2DBlockIOINTELInst : public SPIRVInstTemplateBase { public: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_2d_block_io; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroup2DBlockIOINTEL); } }; class SPIRVSubgroup2DBlockLoadTransposeINTELInst : public SPIRVSubgroup2DBlockIOINTELInst { SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroup2DBlockTransposeINTEL); } }; class SPIRVSubgroup2DBlockLoadTransformINTELInst : public SPIRVSubgroup2DBlockIOINTELInst { SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroup2DBlockTransformINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(Subgroup2DBlockLoad, false, 11) _SPIRV_OP(Subgroup2DBlockPrefetch, false, 10) _SPIRV_OP(Subgroup2DBlockStore, false, 11) #undef _SPIRV_OP #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(Subgroup2DBlockLoadTranspose, false, 11) #undef _SPIRV_OP #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(Subgroup2DBlockLoadTransform, false, 11) #undef _SPIRV_OP class SPIRVSubgroupMatrixMultiplyAccumulateINTELInst : public SPIRVInstTemplateBase { public: llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_subgroup_matrix_multiply_accumulate; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupMatrixMultiplyAccumulateINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(SubgroupMatrixMultiplyAccumulate, true, 7, true, 4) #undef _SPIRV_OP class SPIRVTernaryBitwiseFunctionINTELInst : public SPIRVInstTemplateBase { public: void validate() const override { SPIRVInstruction::validate(); SPIRVErrorLog &SPVErrLog = this->getModule()->getErrorLog(); std::string InstName = "BitwiseFunctionINTEL"; const SPIRVType *ResTy = this->getType(); SPVErrLog.checkError( ResTy->isTypeInt() || (ResTy->isTypeVector() && ResTy->getVectorComponentType()->isTypeInt()), SPIRVEC_InvalidInstruction, InstName + "\nResult type must be an integer scalar or vector.\n"); auto CommonArgCheck = [this, ResTy, &InstName, &SPVErrLog](size_t ArgI, const char *ArgPlacement) { SPIRVValue *Arg = const_cast(this)->getOperand( ArgI); SPVErrLog.checkError( Arg->getType() == ResTy, SPIRVEC_InvalidInstruction, InstName + "\n" + ArgPlacement + " argument must be the same as the result type.\n"); }; CommonArgCheck(0, "First"); CommonArgCheck(1, "Second"); CommonArgCheck(2, "Third"); SPIRVValue *LUTIndexArg = const_cast(this)->getOperand(3); const SPIRVType *LUTIndexArgTy = LUTIndexArg->getType(); SPVErrLog.checkError( LUTIndexArgTy->isTypeInt(32), SPIRVEC_InvalidInstruction, InstName + "\nFourth argument must be a 32-bit integer scalar.\n"); SPVErrLog.checkError( isConstantOpCode(LUTIndexArg->getOpCode()), SPIRVEC_InvalidInstruction, InstName + "\nFourth argument must be constant instruction.\n"); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_ternary_bitwise_function; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityTernaryBitwiseFunctionINTEL); } }; #define _SPIRV_OP(x, ...) \ typedef SPIRVInstTemplate \ SPIRV##x##INTEL; _SPIRV_OP(BitwiseFunction, true, 7) #undef _SPIRV_OP } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVINSTRUCTION_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h000066400000000000000000000223121506646577400237330ustar00rootroot00000000000000//===- SPIRVIsValidEnum.h - SPIR-V isValid enums ----------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V isValid enums. /// //===----------------------------------------------------------------------===// // WARNING: // // This file has been generated using `tools/spirv-tool/gen_spirv.bash` and // should not be modified manually. If the file needs to be updated, edit the // script and any other source file instead, before re-generating this file. //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVISVALIDENUM_H #define SPIRV_LIBSPIRV_SPIRVISVALIDENUM_H #include "SPIRVEnum.h" #include "spirv/unified1/spirv.hpp" #include "spirv_internal.hpp" using namespace spv; namespace SPIRV { inline bool isValid(spv::ExecutionModel V) { switch (static_cast(V)) { case ExecutionModelVertex: case ExecutionModelTessellationControl: case ExecutionModelTessellationEvaluation: case ExecutionModelGeometry: case ExecutionModelFragment: case ExecutionModelGLCompute: case ExecutionModelKernel: case ExecutionModelTaskNV: case ExecutionModelMeshNV: case ExecutionModelRayGenerationKHR: case ExecutionModelIntersectionKHR: case ExecutionModelAnyHitKHR: case ExecutionModelClosestHitKHR: case ExecutionModelMissKHR: case ExecutionModelCallableKHR: case ExecutionModeStreamingInterfaceINTEL: case ExecutionModeMaximumRegistersINTEL: case ExecutionModeMaximumRegistersIdINTEL: case ExecutionModeNamedMaximumRegistersINTEL: return true; default: return false; } } inline bool isValid(spv::AddressingModel V) { switch (V) { case AddressingModelLogical: case AddressingModelPhysical32: case AddressingModelPhysical64: case AddressingModelPhysicalStorageBuffer64: return true; default: return false; } } inline bool isValid(spv::MemoryModel V) { switch (V) { case MemoryModelSimple: case MemoryModelGLSL450: case MemoryModelOpenCL: case MemoryModelVulkan: return true; default: return false; } } inline bool isValid(spv::StorageClass V) { switch (V) { case StorageClassUniformConstant: case StorageClassInput: case StorageClassUniform: case StorageClassOutput: case StorageClassWorkgroup: case StorageClassCrossWorkgroup: case StorageClassPrivate: case StorageClassFunction: case StorageClassGeneric: case StorageClassPushConstant: case StorageClassAtomicCounter: case StorageClassImage: case StorageClassStorageBuffer: case StorageClassCallableDataKHR: case StorageClassIncomingCallableDataKHR: case StorageClassRayPayloadKHR: case StorageClassHitAttributeKHR: case StorageClassIncomingRayPayloadKHR: case StorageClassShaderRecordBufferKHR: case StorageClassPhysicalStorageBuffer: case StorageClassCodeSectionINTEL: case StorageClassDeviceOnlyINTEL: case StorageClassHostOnlyINTEL: return true; default: return false; } } inline bool isValid(spv::LinkageType V) { int LT = V; switch (LT) { case LinkageTypeExport: case LinkageTypeImport: case LinkageTypeLinkOnceODR: case internal::LinkageTypeInternal: return true; default: return false; } } inline bool isValid(spv::AccessQualifier V) { switch (V) { case AccessQualifierReadOnly: case AccessQualifierWriteOnly: case AccessQualifierReadWrite: return true; default: return false; } } inline bool isValid(spv::FunctionParameterAttribute V) { switch (V) { case FunctionParameterAttributeZext: case FunctionParameterAttributeSext: case FunctionParameterAttributeByVal: case FunctionParameterAttributeSret: case FunctionParameterAttributeNoAlias: case FunctionParameterAttributeNoCapture: case FunctionParameterAttributeNoWrite: case FunctionParameterAttributeNoReadWrite: case FunctionParameterAttributeRuntimeAlignedINTEL: return true; default: return false; } } inline bool isValid(spv::BuiltIn V) { switch (static_cast(V)) { case BuiltInPosition: case BuiltInPointSize: case BuiltInClipDistance: case BuiltInCullDistance: case BuiltInVertexId: case BuiltInInstanceId: case BuiltInPrimitiveId: case BuiltInInvocationId: case BuiltInLayer: case BuiltInViewportIndex: case BuiltInTessLevelOuter: case BuiltInTessLevelInner: case BuiltInTessCoord: case BuiltInPatchVertices: case BuiltInFragCoord: case BuiltInPointCoord: case BuiltInFrontFacing: case BuiltInSampleId: case BuiltInSamplePosition: case BuiltInSampleMask: case BuiltInFragDepth: case BuiltInHelperInvocation: case BuiltInNumWorkgroups: case BuiltInWorkgroupSize: case BuiltInWorkgroupId: case BuiltInLocalInvocationId: case BuiltInGlobalInvocationId: case BuiltInLocalInvocationIndex: case BuiltInWorkDim: case BuiltInGlobalSize: case BuiltInEnqueuedWorkgroupSize: case BuiltInGlobalOffset: case BuiltInGlobalLinearId: case BuiltInSubgroupSize: case BuiltInSubgroupMaxSize: case BuiltInNumSubgroups: case BuiltInNumEnqueuedSubgroups: case BuiltInSubgroupId: case BuiltInSubgroupLocalInvocationId: case BuiltInVertexIndex: case BuiltInInstanceIndex: case BuiltInSubgroupEqMask: case BuiltInSubgroupGeMask: case BuiltInSubgroupGtMask: case BuiltInSubgroupLeMask: case BuiltInSubgroupLtMask: case BuiltInBaseVertex: case BuiltInBaseInstance: case BuiltInDrawIndex: case BuiltInPrimitiveShadingRateKHR: case BuiltInDeviceIndex: case BuiltInViewIndex: case BuiltInShadingRateKHR: case BuiltInBaryCoordNoPerspAMD: case BuiltInBaryCoordNoPerspCentroidAMD: case BuiltInBaryCoordNoPerspSampleAMD: case BuiltInBaryCoordSmoothAMD: case BuiltInBaryCoordSmoothCentroidAMD: case BuiltInBaryCoordSmoothSampleAMD: case BuiltInBaryCoordPullModelAMD: case BuiltInFragStencilRefEXT: case BuiltInViewportMaskNV: case BuiltInSecondaryPositionNV: case BuiltInSecondaryViewportMaskNV: case BuiltInPositionPerViewNV: case BuiltInViewportMaskPerViewNV: case BuiltInFullyCoveredEXT: case BuiltInTaskCountNV: case BuiltInPrimitiveCountNV: case BuiltInPrimitiveIndicesNV: case BuiltInClipDistancePerViewNV: case BuiltInCullDistancePerViewNV: case BuiltInLayerPerViewNV: case BuiltInMeshViewCountNV: case BuiltInMeshViewIndicesNV: case BuiltInBaryCoordKHR: case BuiltInBaryCoordNoPerspKHR: case BuiltInFragSizeEXT: case BuiltInFragInvocationCountEXT: case BuiltInLaunchIdKHR: case BuiltInLaunchSizeKHR: case BuiltInWorldRayOriginKHR: case BuiltInWorldRayDirectionKHR: case BuiltInObjectRayOriginKHR: case BuiltInObjectRayDirectionKHR: case BuiltInRayTminKHR: case BuiltInRayTmaxKHR: case BuiltInInstanceCustomIndexKHR: case BuiltInObjectToWorldKHR: case BuiltInWorldToObjectKHR: case BuiltInHitTNV: case BuiltInHitKindKHR: case BuiltInCurrentRayTimeNV: case BuiltInIncomingRayFlagsKHR: case BuiltInRayGeometryIndexKHR: case BuiltInWarpsPerSMNV: case BuiltInSMCountNV: case BuiltInWarpIDNV: case BuiltInSMIDNV: case BuiltInCullMaskKHR: case internal::BuiltInSubDeviceIDINTEL: case internal::BuiltInGlobalHWThreadIDINTEL: return true; default: return false; } } inline bool isValidFunctionControlMask(SPIRVWord Mask) { SPIRVWord ValidMask = 0u; ValidMask |= FunctionControlInlineMask; ValidMask |= FunctionControlDontInlineMask; ValidMask |= FunctionControlPureMask; ValidMask |= FunctionControlConstMask; ValidMask |= FunctionControlOptNoneEXTMask; return (Mask & ~ValidMask) == 0; } } /* namespace SPIRV */ #endif // SPIRV_LIBSPIRV_SPIRVISVALIDENUM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVLLVMUtil.h000066400000000000000000000022521506646577400231640ustar00rootroot00000000000000//===- SPIRVLLVMUtil.h - SPIR-V LLVM-specific Utility Functions -*- C++ -*-===// // // Has inclusions from the LLVM Project, under the Apache License v2.0 with LLVM // Exceptions. See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file /// /// This file defines utility functions dedicated to processing LLVM classes /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVLLVMUTIL_H #define SPIRV_LIBSPIRV_SPIRVLLVMUTIL_H #include "llvm/IR/Constants.h" #include "llvm/Support/Casting.h" namespace SPIRV { inline bool isManifestConstant(const llvm::Constant *C) { if (llvm::isa(C)) { return true; } else if (llvm::isa(C) || llvm::isa(C)) { for (const llvm::Value *Subc : C->operand_values()) { if (!isManifestConstant(llvm::cast(Subc))) return false; } return true; } return false; } } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVLLVMUTIL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVMemAliasingINTEL.h000066400000000000000000000073541506646577400245460ustar00rootroot00000000000000//===- SPIRVMemAliasingINTEL.h - --*- C++ -*-===// // // The LLVM/SPIR-V Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2021 Intel Corporation. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Intel Corporation, nor the names of its contributors // may be used to endorse or promote products derived from this Software without // specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the memory aliasing entries defined in SPIRV spec with op /// codes. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVMEMALIASINGINTEL_H #define SPIRV_LIBSPIRV_SPIRVMEMALIASINGINTEL_H #include "SPIRVEntry.h" namespace SPIRV { template class SPIRVMemAliasingINTELGeneric : public SPIRVEntry { public: SPIRVMemAliasingINTELGeneric(SPIRVModule *TheModule, SPIRVId TheId, const std::vector &TheArgs) : SPIRVEntry(TheModule, TheArgs.size() + TheFixedWordCount, TheOpCode, TheId), Args(TheArgs) { SPIRVMemAliasingINTELGeneric::validate(); assert(TheModule && "Invalid module"); } SPIRVMemAliasingINTELGeneric() : SPIRVEntry(TheOpCode) {} const std::vector &getArguments() const { return Args; } void setWordCount(SPIRVWord TheWordCount) override { SPIRVEntry::setWordCount(TheWordCount); Args.resize(TheWordCount - TheFixedWordCount); } void validate() const override { SPIRVEntry::validate(); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityMemoryAccessAliasingINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_memory_access_aliasing; } protected: static const SPIRVWord FixedWC = TheFixedWordCount; static const Op OC = TheOpCode; std::vector Args; _SPIRV_DEF_ENCDEC2(Id, Args) }; #define _SPIRV_OP(x, ...) \ typedef SPIRVMemAliasingINTELGeneric SPIRV##x; // Intel Memory Alasing Instructions _SPIRV_OP(AliasDomainDeclINTEL, 2) _SPIRV_OP(AliasScopeDeclINTEL, 2) _SPIRV_OP(AliasScopeListDeclINTEL, 2) #undef _SPIRV_OP } // SPIRV #endif // SPIRV_LIBSPIRV_SPIRVMEMALIASINGINTEL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVModule.cpp000066400000000000000000003060241506646577400233400ustar00rootroot00000000000000//===- SPIRVModule.cpp - Class to represent SPIR-V module -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements Module class for SPIR-V. /// //===----------------------------------------------------------------------===// #include "SPIRVModule.h" #include "SPIRVAsm.h" #include "SPIRVDebug.h" #include "SPIRVEntry.h" #include "SPIRVEnum.h" #include "SPIRVFnVar.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVMemAliasingINTEL.h" #include "SPIRVNameMapEnum.h" #include "SPIRVOpCode.h" #include "SPIRVStream.h" #include "SPIRVType.h" #include "SPIRVValue.h" #include "llvm/ADT/APInt.h" #include #include #include namespace SPIRV { namespace { std::string to_string(uint32_t Version) { std::string Res; switch (Version) { case static_cast(VersionNumber::SPIRV_1_0): Res = "1.0"; break; case static_cast(VersionNumber::SPIRV_1_1): Res = "1.1"; break; case static_cast(VersionNumber::SPIRV_1_2): Res = "1.2"; break; case static_cast(VersionNumber::SPIRV_1_3): Res = "1.3"; break; case static_cast(VersionNumber::SPIRV_1_4): Res = "1.4"; break; case static_cast(VersionNumber::SPIRV_1_5): Res = "1.5"; break; case static_cast(VersionNumber::SPIRV_1_6): Res = "1.6"; break; default: Res = "unknown"; } Res += " (" + std::to_string(Version) + ")"; return Res; } std::string to_string(VersionNumber Version) { return to_string(static_cast(Version)); } } // Anonymous namespace SPIRVModule::SPIRVModule() : AutoAddCapability(true), ValidateCapability(false), IsValid(true) {} SPIRVModule::~SPIRVModule() {} class SPIRVModuleImpl : public SPIRVModule { public: SPIRVModuleImpl() : SPIRVModule(), NextId(1), SPIRVVersion(static_cast(VersionNumber::SPIRV_1_0)), GeneratorId(SPIRVGEN_KhronosLLVMSPIRVTranslator), GeneratorVer(0), InstSchema(SPIRVISCH_Default), SrcLang(SourceLanguageOpenCL_C), SrcLangVer(102000) { AddrModel = sizeof(size_t) == 32 ? AddressingModelPhysical32 : AddressingModelPhysical64; // OpenCL memory model requires Kernel capability setMemoryModel(MemoryModelOpenCL); } SPIRVModuleImpl(const SPIRV::TranslatorOpts &Opts) : SPIRVModuleImpl() { TranslationOpts = Opts; } ~SPIRVModuleImpl() override; // Object query functions bool exist(SPIRVId) const override; bool exist(SPIRVId, SPIRVEntry **) const override; SPIRVId getId(SPIRVId Id = SPIRVID_INVALID, unsigned Increment = 1); SPIRVEntry *getEntry(SPIRVId Id) const override; // If we have at least on OpLine in the module the CurrentLine is non-empty bool hasDebugInfo() const override { return CurrentLine.get() || !DebugInstVec.empty(); } // Error handling functions SPIRVErrorLog &getErrorLog() override { return ErrLog; } SPIRVErrorCode getError(std::string &ErrMsg) override { return ErrLog.getError(ErrMsg); } bool checkExtension(ExtensionID Ext, SPIRVErrorCode ErrCode, const std::string &Msg) override { if (ErrLog.checkError(isAllowedToUseExtension(Ext), ErrCode, Msg)) return true; setInvalid(); return false; } // Module query functions SPIRVAddressingModelKind getAddressingModel() override { return AddrModel; } SPIRVExtInstSetKind getBuiltinSet(SPIRVId SetId) const override; const SPIRVCapMap &getCapability() const override { return CapMap; } const SPIRVConditionalCapMap &getConditionalCapabilities() const override { return ConditionalCapMap; } const SPIRVConditionalEntryPointVec & getConditionalEntryPoints() const override { return ConditionalEntryPointVec; } bool hasCapability(SPIRVCapabilityKind Cap) const override { return CapMap.find(Cap) != CapMap.end(); } std::set &getExtension() override { return SPIRVExt; } SPIRVConditionalExtensionSet &getConditionalExtensions() override { return SPIRVCondExt; } SPIRVFunction *getFunction(unsigned I) const override { return FuncVec[I]; } SPIRVVariable *getVariable(unsigned I) const override { return VariableVec[I]; } SPIRVValue *getConst(unsigned I) const override { return ConstVec[I]; } std::vector *getDecorateVec() override { return &DecorateVec; } std::vector *getFuncVec() override { return &FuncVec; } SPIRVValue *getValue(SPIRVId TheId) const override; std::vector getValues(const std::vector &) const override; std::vector getIds(const std::vector &) const override; std::vector getIds(const std::vector &) const override; SPIRVType *getValueType(SPIRVId TheId) const override; std::vector getValueTypes(const std::vector &) const override; SPIRVMemoryModelKind getMemoryModel() const override { return MemoryModel; } SPIRVConstant *getLiteralAsConstant(unsigned Literal) override; unsigned getNumFunctions() const override { return FuncVec.size(); } unsigned getNumVariables() const override { return VariableVec.size(); } unsigned getNumConsts() const override { return ConstVec.size(); } std::vector getFunctionPointers() const override { std::vector Res; for (auto *C : ConstVec) if (C->getOpCode() == OpConstantFunctionPointerINTEL) Res.emplace_back(C); return Res; } SourceLanguage getSourceLanguage(SPIRVWord *Ver = nullptr) const override { if (Ver) *Ver = SrcLangVer; return SrcLang; } std::set &getSourceExtension() override { return SrcExtension; } bool isEntryPoint(SPIRVExecutionModelKind, SPIRVId EP) const override; unsigned short getGeneratorId() const override { return GeneratorId; } unsigned short getGeneratorVer() const override { return GeneratorVer; } SPIRVWord getSPIRVVersion() const override { return SPIRVVersion; } const std::vector &getDebugInstVec() const override { return DebugInstVec; } const std::vector &getAuxDataInstVec() const override { return AuxDataInstVec; } const std::vector &getStringVec() const override { return StringVec; } // Module changing functions bool importBuiltinSet(const std::string &, SPIRVId *) override; bool importBuiltinSetWithId(const std::string &, SPIRVId) override; void setAddressingModel(SPIRVAddressingModelKind AM) override { AddrModel = AM; } void setAlignment(SPIRVValue *, SPIRVWord) override; void setMemoryModel(SPIRVMemoryModelKind MM) override { MemoryModel = MM; if (MemoryModel == spv::MemoryModelOpenCL) addCapability(CapabilityKernel); } void setName(SPIRVEntry *E, const std::string &Name) override; void setSourceLanguage(SourceLanguage Lang, SPIRVWord Ver) override { SrcLang = Lang; SrcLangVer = Ver; } void setGeneratorId(unsigned short Id) override { GeneratorId = Id; } void setGeneratorVer(unsigned short Ver) override { GeneratorVer = Ver; } void resolveUnknownStructFields() override; void insertEntryNoId(SPIRVEntry *Entry) override { EntryNoId.insert(Entry); } void setSPIRVVersion(SPIRVWord Ver) override { if (!this->isAllowedToUseVersion(static_cast(Ver))) { std::stringstream SS; SS << "SPIR-V version was restricted to at most " << to_string(getMaximumAllowedSPIRVVersion()) << " but a construct from the input requires SPIR-V version " << to_string(Ver) << " or above\n"; getErrorLog().checkError(false, SPIRVEC_RequiresVersion, SS.str()); setInvalid(); return; } SPIRVVersion = Ver; } bool eraseReferencesOfInst(SPIRVId Id) override { const auto *const Entry = getEntry(Id); if (!Entry) return false; if (!Entry->hasId()) return false; // Remove all OpNames referencing the Id if (NamedId.find(Id) != NamedId.end()) NamedId.erase(Id); // Remove all OpMemberNames referencing the Id erase_if(MemberNameVec, [Id](auto M) { return M->getTargetId() == Id; }); // Remove all decorations of the Id erase_if(DecorateVec, [Id](auto D) { return D->getTargetId() == Id; }); // If Id points to a function, remove its OpEntryPoint, OpExecutionMode // and OpExecutionModeId if (Entry->getOpCode() == OpFunction) erase_if(EntryPointVec, [Id](auto EP) { return EP->getTargetId() == Id; }); return true; } void eraseCapability(SPIRVCapabilityKind CapKind) override { CapMap.erase(CapKind); } // Object creation functions template void addTo(std::vector &V, SPIRVEntry *E); SPIRVEntry *addEntry(SPIRVEntry *E) override; SPIRVBasicBlock *addBasicBlock(SPIRVFunction *, SPIRVId) override; SPIRVString *getString(const std::string &Str) override; SPIRVMemberName *addMemberName(SPIRVTypeStruct *ST, SPIRVWord MemberNumber, const std::string &Name) override; void addUnknownStructField(SPIRVTypeStruct *Struct, unsigned I, SPIRVId ID) override; void addLine(SPIRVEntry *E, SPIRVId FileNameId, SPIRVWord Line, SPIRVWord Column) override; const std::shared_ptr &getCurrentLine() const override; void setCurrentLine(const std::shared_ptr &Line) override; void addDebugLine(SPIRVEntry *E, SPIRVType *TheType, SPIRVId FileNameId, SPIRVWord LineStart, SPIRVWord LineEnd, SPIRVWord ColumnStart, SPIRVWord ColumnEnd) override; const std::shared_ptr & getCurrentDebugLine() const override; void setCurrentDebugLine( const std::shared_ptr &DebugLine) override; void addCapability(SPIRVCapabilityKind) override; void addCapabilityInternal(SPIRVCapabilityKind) override; void addConditionalCapability(SPIRVId, SPIRVCapabilityKind) override; void eraseConditionalCapability(SPIRVId, SPIRVCapabilityKind) override; void addExtension(ExtensionID) override; const SPIRVDecorateGeneric *addDecorate(SPIRVDecorateGeneric *) override; SPIRVDecorationGroup *addDecorationGroup() override; SPIRVDecorationGroup * addDecorationGroup(SPIRVDecorationGroup *Group) override; SPIRVGroupDecorate * addGroupDecorate(SPIRVDecorationGroup *Group, const std::vector &Targets) override; SPIRVGroupDecorateGeneric * addGroupDecorateGeneric(SPIRVGroupDecorateGeneric *GDec) override; SPIRVGroupMemberDecorate * addGroupMemberDecorate(SPIRVDecorationGroup *Group, const std::vector &Targets) override; void addEntryPoint(SPIRVExecutionModelKind ExecModel, SPIRVId EntryPoint, const std::string &Name, const std::vector &Variables) override; void addConditionalEntryPoint(SPIRVId, SPIRVExecutionModelKind ExecModel, SPIRVId EntryPoint, const std::string &Name, const std::vector &Variables) override; void specializeConditionalEntryPoints(SPIRVId, bool) override; SPIRVForward *addForward(SPIRVType *Ty) override; SPIRVForward *addForward(SPIRVId, SPIRVType *Ty) override; SPIRVFunction *addFunction(SPIRVFunction *) override; SPIRVFunction *addFunction(SPIRVTypeFunction *, SPIRVId) override; SPIRVEntry *replaceForward(SPIRVForward *, SPIRVEntry *) override; void eraseInstruction(SPIRVInstruction *, SPIRVBasicBlock *) override; bool eraseValue(SPIRVValue *) override; // Type creation functions template T *addType(T *Ty); SPIRVTypeArray *addArrayType(SPIRVType *, SPIRVConstant *) override; SPIRVTypeBool *addBoolType() override; SPIRVTypeFloat *addFloatType(unsigned BitWidth, unsigned FloatingPointEncoding) override; SPIRVTypeFunction *addFunctionType(SPIRVType *, const std::vector &) override; SPIRVTypeInt *addIntegerType(unsigned BitWidth) override; SPIRVTypeOpaque *addOpaqueType(const std::string &) override; SPIRVTypePointer *addPointerType(SPIRVStorageClassKind, SPIRVType *) override; SPIRVTypeImage *addImageType(SPIRVType *, const SPIRVTypeImageDescriptor &) override; SPIRVTypeImage *addImageType(SPIRVType *, const SPIRVTypeImageDescriptor &, SPIRVAccessQualifierKind) override; SPIRVTypeSampler *addSamplerType() override; SPIRVTypePipeStorage *addPipeStorageType() override; SPIRVTypeSampledImage *addSampledImageType(SPIRVTypeImage *T) override; SPIRVTypeStruct *openStructType(unsigned, const std::string &) override; SPIRVEntry *addTypeStructContinuedINTEL(unsigned NumMembers) override; void closeStructType(SPIRVTypeStruct *T, bool) override; SPIRVTypeVector *addVectorType(SPIRVType *, SPIRVWord) override; SPIRVTypeJointMatrixINTEL * addJointMatrixINTELType(SPIRVType *, std::vector) override; SPIRVTypeCooperativeMatrixKHR * addCooperativeMatrixKHRType(SPIRVType *, std::vector) override; SPIRVInstruction * addCooperativeMatrixLengthKHRInst(SPIRVType *, SPIRVType *, SPIRVBasicBlock *) override; SPIRVType *addOpaqueGenericType(Op) override; SPIRVTypeDeviceEvent *addDeviceEventType() override; SPIRVTypeQueue *addQueueType() override; SPIRVTypePipe *addPipeType() override; SPIRVTypeVoid *addVoidType() override; SPIRVType *addSubgroupAvcINTELType(Op) override; SPIRVTypeVmeImageINTEL *addVmeImageINTELType(SPIRVTypeImage *T) override; SPIRVTypeBufferSurfaceINTEL * addBufferSurfaceINTELType(SPIRVAccessQualifierKind Access) override; SPIRVTypeTokenINTEL *addTokenTypeINTEL() override; // Constant creation functions SPIRVInstruction *addBranchInst(SPIRVLabel *, SPIRVBasicBlock *) override; SPIRVInstruction *addBranchConditionalInst(SPIRVValue *, SPIRVLabel *, SPIRVLabel *, SPIRVBasicBlock *) override; SPIRVValue *addCompositeConstant(SPIRVType *, const std::vector &) override; SPIRVEntry *addCompositeConstantContinuedINTEL( const std::vector &) override; SPIRVValue * addSpecConstantComposite(SPIRVType *Ty, const std::vector &Elements) override; SPIRVEntry *addSpecConstantCompositeContinuedINTEL( const std::vector &) override; SPIRVValue *addConstantFunctionPointerINTEL(SPIRVType *Ty, SPIRVFunction *F) override; SPIRVValue *addConstant(SPIRVValue *) override; SPIRVValue *addConstant(SPIRVType *, uint64_t) override; SPIRVValue *addConstant(SPIRVType *, llvm::APInt) override; SPIRVValue *addSpecConstant(SPIRVType *, uint64_t) override; SPIRVValue *addDoubleConstant(SPIRVTypeFloat *, double) override; SPIRVValue *addFloatConstant(SPIRVTypeFloat *, float) override; SPIRVValue *addIntegerConstant(SPIRVTypeInt *, uint64_t) override; SPIRVValue *addNullConstant(SPIRVType *) override; SPIRVValue *addUndef(SPIRVType *TheType) override; SPIRVValue *addSamplerConstant(SPIRVType *TheType, SPIRVWord AddrMode, SPIRVWord ParametricMode, SPIRVWord FilterMode) override; SPIRVValue *addPipeStorageConstant(SPIRVType *TheType, SPIRVWord PacketSize, SPIRVWord PacketAlign, SPIRVWord Capacity) override; // Instruction creation functions SPIRVInstruction *addPtrAccessChainInst(SPIRVType *, SPIRVValue *, std::vector, SPIRVBasicBlock *, bool) override; SPIRVInstruction *addAsyncGroupCopy(SPIRVValue *Scope, SPIRVValue *Dest, SPIRVValue *Src, SPIRVValue *NumElems, SPIRVValue *Stride, SPIRVValue *Event, SPIRVBasicBlock *BB) override; SPIRVInstruction *addExtInst(SPIRVType *, SPIRVWord, SPIRVWord, const std::vector &, SPIRVBasicBlock *, SPIRVInstruction * = nullptr) override; SPIRVInstruction *addExtInst(SPIRVType *, SPIRVWord, SPIRVWord, const std::vector &, SPIRVBasicBlock *, SPIRVInstruction * = nullptr) override; SPIRVEntry *createDebugInfo(SPIRVWord, SPIRVType *TheType, const std::vector &) override; SPIRVEntry *addDebugInfo(SPIRVWord, SPIRVType *TheType, const std::vector &) override; SPIRVEntry *addAuxData(SPIRVWord, SPIRVType *TheType, const std::vector &) override; SPIRVEntry *addModuleProcessed(const std::string &) override; std::vector getModuleProcessedVec() override; SPIRVInstruction *addBinaryInst(Op, SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addCallInst(SPIRVFunction *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addIndirectCallInst(SPIRVValue *, SPIRVType *, const std::vector &, SPIRVBasicBlock *) override; SPIRVEntry *getOrAddAsmTargetINTEL(const std::string &) override; SPIRVValue *addAsmINTEL(SPIRVTypeFunction *, SPIRVAsmTargetINTEL *, const std::string &, const std::string &) override; SPIRVInstruction *addAsmCallINTELInst(SPIRVAsmINTEL *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addCmpInst(Op, SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addLoadInst(SPIRVValue *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addPhiInst(SPIRVType *, std::vector, SPIRVBasicBlock *) override; SPIRVInstruction *addCompositeConstructInst(SPIRVType *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addCompositeExtractInst(SPIRVType *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction * addCompositeInsertInst(SPIRVValue *Object, SPIRVValue *Composite, const std::vector &Indices, SPIRVBasicBlock *BB) override; SPIRVInstruction *addCopyObjectInst(SPIRVType *TheType, SPIRVValue *Operand, SPIRVBasicBlock *BB) override; SPIRVInstruction *addCopyMemoryInst(SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addCopyMemorySizedInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addControlBarrierInst(SPIRVValue *ExecKind, SPIRVValue *MemKind, SPIRVValue *MemSema, SPIRVBasicBlock *BB) override; SPIRVInstruction *addGroupInst(Op OpCode, SPIRVType *Type, Scope Scope, const std::vector &Ops, SPIRVBasicBlock *BB) override; virtual SPIRVInstruction * addInstruction(SPIRVInstruction *Inst, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore = nullptr); SPIRVInstTemplateBase *addInstTemplate(Op OC, SPIRVBasicBlock *BB, SPIRVType *Ty) override; SPIRVInstTemplateBase *addInstTemplate(Op OC, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) override; void addInstTemplate(SPIRVInstTemplateBase *Ins, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) override; SPIRVInstruction *addLifetimeInst(Op OC, SPIRVValue *Object, SPIRVWord Size, SPIRVBasicBlock *BB) override; SPIRVInstruction *addMemoryBarrierInst(Scope ScopeKind, SPIRVWord MemFlag, SPIRVBasicBlock *BB) override; SPIRVInstruction *addUnreachableInst(SPIRVBasicBlock *) override; SPIRVInstruction *addReturnInst(SPIRVBasicBlock *) override; SPIRVInstruction *addReturnValueInst(SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addSelectInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction * addLoopMergeInst(SPIRVId MergeBlock, SPIRVId ContinueTarget, SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) override; SPIRVInstruction * addLoopControlINTELInst(SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) override; SPIRVInstruction *addFixedPointIntelInst(Op OC, SPIRVType *ResTy, SPIRVValue *Input, const std::vector &Ops, SPIRVBasicBlock *BB) override; SPIRVInstruction *addArbFloatPointIntelInst(Op OC, SPIRVType *ResTy, SPIRVValue *InA, SPIRVValue *InB, const std::vector &Ops, SPIRVBasicBlock *BB) override; SPIRVInstruction *addSelectionMergeInst(SPIRVId MergeBlock, SPIRVWord SelectionControl, SPIRVBasicBlock *BB) override; SPIRVInstruction *addStoreInst(SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) override; SPIRVInstruction *addSwitchInst( SPIRVValue *, SPIRVBasicBlock *, const std::vector, SPIRVBasicBlock *>> &, SPIRVBasicBlock *) override; SPIRVInstruction *addVectorTimesScalarInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheScalar, SPIRVBasicBlock *BB) override; SPIRVInstruction *addVectorTimesMatrixInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheScalar, SPIRVBasicBlock *BB) override; SPIRVInstruction *addMatrixTimesScalarInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheScalar, SPIRVBasicBlock *BB) override; SPIRVInstruction *addMatrixTimesVectorInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheVector, SPIRVBasicBlock *BB) override; SPIRVInstruction *addMatrixTimesMatrixInst(SPIRVType *TheType, SPIRVId M1, SPIRVId M2, SPIRVBasicBlock *BB) override; SPIRVInstruction *addTransposeInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVBasicBlock *BB) override; SPIRVInstruction *addUnaryInst(Op, SPIRVType *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addVariable(SPIRVType *, bool, SPIRVLinkageTypeKind, SPIRVValue *, const std::string &, SPIRVStorageClassKind, SPIRVBasicBlock *) override; SPIRVValue *addVectorShuffleInst(SPIRVType *Type, SPIRVValue *Vec1, SPIRVValue *Vec2, const std::vector &Components, SPIRVBasicBlock *BB) override; SPIRVInstruction *addVectorExtractDynamicInst(SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addVectorInsertDynamicInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addFPGARegINTELInst(SPIRVType *, SPIRVValue *, SPIRVBasicBlock *) override; SPIRVInstruction *addSampledImageInst(SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) override; template SPIRVEntry *getOrAddMemAliasingINTELInst(std::vector Args, llvm::MDNode *MD); SPIRVEntry *getOrAddAliasDomainDeclINTELInst(std::vector Args, llvm::MDNode *MD) override; SPIRVEntry *getOrAddAliasScopeDeclINTELInst(std::vector Args, llvm::MDNode *MD) override; SPIRVEntry *getOrAddAliasScopeListDeclINTELInst(std::vector Args, llvm::MDNode *MD) override; SPIRVInstruction *addAssumeTrueKHRInst(SPIRVValue *Condition, SPIRVBasicBlock *BB) override; SPIRVInstruction *addExpectKHRInst(SPIRVType *ResultTy, SPIRVValue *Value, SPIRVValue *ExpectedValue, SPIRVBasicBlock *BB) override; virtual SPIRVId getExtInstSetId(SPIRVExtInstSetKind Kind) const override; // I/O functions friend spv_ostream &operator<<(spv_ostream &O, SPIRVModule &M); friend std::istream &operator>>(std::istream &I, SPIRVModule &M); private: SPIRVErrorLog ErrLog; SPIRVId NextId; SPIRVWord SPIRVVersion; unsigned short GeneratorId; unsigned short GeneratorVer; SPIRVInstructionSchemaKind InstSchema; SourceLanguage SrcLang; SPIRVWord SrcLangVer; std::set SrcExtension; std::set SPIRVExt; SPIRVConditionalExtensionSet SPIRVCondExt; SPIRVAddressingModelKind AddrModel; SPIRVMemoryModelKind MemoryModel; typedef std::map SPIRVIdToEntryMap; typedef std::set SPIRVEntrySet; typedef std::set SPIRVIdSet; typedef std::vector SPIRVIdVec; typedef std::vector SPIRVFunctionVector; typedef std::vector SPIRVForwardPointerVec; typedef std::vector SPIRVTypeVec; typedef std::vector SPIRVConstantVector; typedef std::vector SPIRVVariableVec; typedef std::vector SPIRVStringVec; typedef std::vector SPIRVMemberNameVec; typedef std::vector SPIRVDecGroupVec; typedef std::vector SPIRVGroupDecVec; typedef std::vector SPIRVAsmTargetVector; typedef std::vector SPIRVAsmVector; typedef std::vector SPIRVEntryPointVec; typedef std::map SPIRVIdToInstructionSetMap; std::map ExtInstSetIds; typedef std::map SPIRVIdToBuiltinSetMap; typedef std::map SPIRVExecModelIdSetMap; typedef std::unordered_map SPIRVStringMap; typedef std::map>> SPIRVUnknownStructFieldMap; typedef std::vector SPIRVAliasInstMDVec; typedef std::unordered_map SPIRVAliasInstMDMap; SPIRVForwardPointerVec ForwardPointerVec; SPIRVTypeVec TypeVec; SPIRVIdToEntryMap IdEntryMap; SPIRVIdToEntryMap IdTypeForwardMap; // Forward declared IDs SPIRVFunctionVector FuncVec; SPIRVConstantVector ConstVec; SPIRVVariableVec VariableVec; SPIRVEntrySet EntryNoId; // Entries without id SPIRVIdToInstructionSetMap IdToInstSetMap; SPIRVIdToBuiltinSetMap IdBuiltinMap; SPIRVIdSet NamedId; SPIRVStringVec StringVec; SPIRVMemberNameVec MemberNameVec; std::shared_ptr CurrentLine; std::shared_ptr CurrentDebugLine; SPIRVDecorateVec DecorateVec; SPIRVDecGroupVec DecGroupVec; SPIRVGroupDecVec GroupDecVec; SPIRVAsmTargetVector AsmTargetVec; SPIRVAsmVector AsmVec; SPIRVExecModelIdSetMap EntryPointSet; SPIRVEntryPointVec EntryPointVec; SPIRVExecModelIdSetMap ConditionalEntryPointSet; SPIRVConditionalEntryPointVec ConditionalEntryPointVec; SPIRVStringMap StrMap; SPIRVCapMap CapMap; SPIRVConditionalCapMap ConditionalCapMap; SPIRVUnknownStructFieldMap UnknownStructFieldMap; std::map IntTypeMap; SmallDenseMap, SPIRVTypeFloat *, 4> FloatTypeMap; std::map LiteralMap; std::vector DebugInstVec; std::vector AuxDataInstVec; std::vector ModuleProcessedVec; SPIRVAliasInstMDVec AliasInstMDVec; SPIRVAliasInstMDMap AliasInstMDMap; void layoutEntry(SPIRVEntry *Entry); }; SPIRVModuleImpl::~SPIRVModuleImpl() { for (auto I : EntryNoId) delete I; for (auto I : IdEntryMap) delete I.second; for (auto C : CapMap) delete C.second; for (auto C : ConditionalCapMap) delete C.second; for (auto *M : ModuleProcessedVec) delete M; } const std::shared_ptr & SPIRVModuleImpl::getCurrentLine() const { return CurrentLine; } void SPIRVModuleImpl::setCurrentLine( const std::shared_ptr &Line) { CurrentLine = Line; } void SPIRVModuleImpl::addLine(SPIRVEntry *E, SPIRVId FileNameId, SPIRVWord Line, SPIRVWord Column) { if (!(CurrentLine && CurrentLine->equals(FileNameId, Line, Column))) CurrentLine.reset(new SPIRVLine(this, FileNameId, Line, Column)); assert(E && "invalid entry"); E->setLine(CurrentLine); } const std::shared_ptr & SPIRVModuleImpl::getCurrentDebugLine() const { return CurrentDebugLine; } void SPIRVModuleImpl::setCurrentDebugLine( const std::shared_ptr &DebugLine) { CurrentDebugLine = DebugLine; } namespace { bool isDebugLineEqual(const SPIRVExtInst &CurrentDebugLine, SPIRVId FileNameId, SPIRVId LineStartId, SPIRVId LineEndId, SPIRVId ColumnStartId, SPIRVId ColumnEndId) { assert(CurrentDebugLine.getExtOp() == SPIRVDebug::DebugLine); const std::vector CurrentDebugLineArgs = CurrentDebugLine.getArguments(); using namespace SPIRVDebug::Operand::DebugLine; return CurrentDebugLineArgs[SourceIdx] == FileNameId && CurrentDebugLineArgs[StartIdx] == LineStartId && CurrentDebugLineArgs[EndIdx] == LineEndId && CurrentDebugLineArgs[ColumnStartIdx] == ColumnStartId && CurrentDebugLineArgs[ColumnEndIdx] == ColumnEndId; } } // namespace void SPIRVModuleImpl::addDebugLine(SPIRVEntry *E, SPIRVType *TheType, SPIRVId FileNameId, SPIRVWord LineStart, SPIRVWord LineEnd, SPIRVWord ColumnStart, SPIRVWord ColumnEnd) { if (!(CurrentDebugLine && isDebugLineEqual(*CurrentDebugLine, FileNameId, getLiteralAsConstant(LineStart)->getId(), getLiteralAsConstant(LineEnd)->getId(), getLiteralAsConstant(ColumnStart)->getId(), getLiteralAsConstant(ColumnEnd)->getId()))) { using namespace SPIRVDebug::Operand::DebugLine; std::vector DebugLineOps(OperandCount); DebugLineOps[SourceIdx] = FileNameId; DebugLineOps[StartIdx] = getLiteralAsConstant(LineStart)->getId(); DebugLineOps[EndIdx] = getLiteralAsConstant(LineEnd)->getId(); DebugLineOps[ColumnStartIdx] = getLiteralAsConstant(ColumnStart)->getId(); DebugLineOps[ColumnEndIdx] = getLiteralAsConstant(ColumnEnd)->getId(); CurrentDebugLine.reset(static_cast( createDebugInfo(SPIRVDebug::DebugLine, TheType, DebugLineOps))); } assert(E && "invalid entry"); E->setDebugLine(CurrentDebugLine); } SPIRVValue *SPIRVModuleImpl::addSamplerConstant(SPIRVType *TheType, SPIRVWord AddrMode, SPIRVWord ParametricMode, SPIRVWord FilterMode) { return addConstant(new SPIRVConstantSampler(this, TheType, getId(), AddrMode, ParametricMode, FilterMode)); } SPIRVValue *SPIRVModuleImpl::addPipeStorageConstant(SPIRVType *TheType, SPIRVWord PacketSize, SPIRVWord PacketAlign, SPIRVWord Capacity) { return addConstant(new SPIRVConstantPipeStorage( this, TheType, getId(), PacketSize, PacketAlign, Capacity)); } void SPIRVModuleImpl::addExtension(ExtensionID Ext) { std::string ExtName; SPIRVMap::find(Ext, &ExtName); if (!getErrorLog().checkError(isAllowedToUseExtension(Ext), SPIRVEC_RequiresExtension, ExtName)) { setInvalid(); return; } SPIRVExt.insert(ExtName); // SPV_EXT_shader_atomic_float16_add extends the // SPV_EXT_shader_atomic_float_add extension. // The specification requires both extensions to be added to use // AtomicFloat16AddEXT capability whereas getRequiredExtension() // is able to return a single extensionID. if (Ext == ExtensionID::SPV_EXT_shader_atomic_float16_add) { SPIRVMap::find( ExtensionID::SPV_EXT_shader_atomic_float_add, &ExtName); SPIRVExt.insert(ExtName); } } void SPIRVModuleImpl::addCapability(SPIRVCapabilityKind Cap) { addCapabilities(SPIRV::getCapability(Cap)); SPIRVDBG(spvdbgs() << "addCapability: " << SPIRVCapabilityNameMap::map(Cap) << '\n'); if (hasCapability(Cap)) return; auto *CapObj = new SPIRVCapability(this, Cap); if (AutoAddExtensions) { // While we are reading existing SPIR-V we need to read it as-is and don't // add required extensions for each entry automatically auto Ext = CapObj->getRequiredExtension(); if (Ext.hasValue()) addExtension(Ext.getValue()); } CapMap.insert(std::make_pair(Cap, CapObj)); } void SPIRVModuleImpl::addCapabilityInternal(SPIRVCapabilityKind Cap) { if (AutoAddCapability) { if (hasCapability(Cap)) return; CapMap.insert(std::make_pair(Cap, new SPIRVCapability(this, Cap))); } } void SPIRVModuleImpl::addConditionalCapability(SPIRVId Condition, SPIRVCapabilityKind Cap) { SPIRVDBG(spvdbgs() << "addConditionalCapability: " << SPIRVCapabilityNameMap::map(Cap) << ", condition: " << Condition << '\n'); if (ConditionalCapMap.find(std::make_pair(Condition, Cap)) != ConditionalCapMap.end()) { return; } auto *CapObj = new SPIRVConditionalCapabilityINTEL(this, Condition, Cap); if (AutoAddExtensions) { assert(false && "Auto adding conditional extensions is not supported."); } ConditionalCapMap.insert( std::make_pair(std::make_pair(Condition, Cap), CapObj)); } void SPIRVModuleImpl::eraseConditionalCapability(SPIRVId Condition, SPIRVCapabilityKind Cap) { ConditionalCapMap.erase(std::make_pair(Condition, Cap)); } SPIRVConstant *SPIRVModuleImpl::getLiteralAsConstant(unsigned Literal) { auto Loc = LiteralMap.find(Literal); if (Loc != LiteralMap.end()) return Loc->second; auto Ty = addIntegerType(32); auto V = new SPIRVConstant(this, Ty, getId(), static_cast(Literal)); LiteralMap[Literal] = V; addConstant(V); return V; } void SPIRVModuleImpl::layoutEntry(SPIRVEntry *E) { auto OC = E->getOpCode(); int IntOC = static_cast(OC); switch (IntOC) { case OpString: addTo(StringVec, E); break; case OpMemberName: addTo(MemberNameVec, E); break; case OpVariable: { auto BV = static_cast(E); if (!BV->getParent()) addTo(VariableVec, E); } break; case OpExtInst: { SPIRVExtInst *EI = static_cast(E); if ((EI->getExtSetKind() == SPIRVEIS_Debug || EI->getExtSetKind() == SPIRVEIS_OpenCL_DebugInfo_100 || EI->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_100 || EI->getExtSetKind() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) && EI->getExtOp() != SPIRVDebug::Declare && EI->getExtOp() != SPIRVDebug::Value && EI->getExtOp() != SPIRVDebug::Scope && EI->getExtOp() != SPIRVDebug::NoScope) { DebugInstVec.push_back(EI); } if (EI->getExtSetKind() == SPIRVEIS_NonSemantic_AuxData) AuxDataInstVec.push_back(EI); break; } case OpAsmTargetINTEL: { addTo(AsmTargetVec, E); break; } case OpAliasDomainDeclINTEL: case OpAliasScopeDeclINTEL: case OpAliasScopeListDeclINTEL: { addTo(AliasInstMDVec, E); break; } case OpAsmINTEL: { addTo(AsmVec, E); break; } default: if (isTypeOpCode(OC)) TypeVec.push_back(static_cast(E)); else if (isConstantOpCode(OC)) ConstVec.push_back(static_cast(E)); break; } } // Add an entry to the id to entry map. // Assert if the id is mapped to a different entry. // Certain entries need to be add to specific collectors to maintain // logic layout of SPIRV. SPIRVEntry *SPIRVModuleImpl::addEntry(SPIRVEntry *Entry) { assert(Entry && "Invalid entry"); if (Entry->hasId()) { SPIRVId Id = Entry->getId(); assert(Entry->getId() != SPIRVID_INVALID && "Invalid id"); SPIRVEntry *Mapped = nullptr; if (exist(Id, &Mapped)) { if (Mapped->getOpCode() == internal::OpForward) { replaceForward(static_cast(Mapped), Entry); } else { assert(Mapped == Entry && "Id used twice"); } } else IdEntryMap[Id] = Entry; } else { // Collect entries with no ID to de-allocate them at the end. // Entry of OpLine will be deleted by std::shared_ptr automatically. if (Entry->getOpCode() != OpLine) EntryNoId.insert(Entry); // Store the known ID of pointer type that would be declared later. if (Entry->getOpCode() == OpTypeForwardPointer) IdTypeForwardMap[static_cast(Entry) ->getPointerId()] = Entry; } Entry->setModule(this); layoutEntry(Entry); if (AutoAddCapability) { for (auto &I : Entry->getRequiredCapability()) { addCapability(I); } } if (ValidateCapability) { assert(none_of( Entry->getRequiredCapability().begin(), Entry->getRequiredCapability().end(), [this](SPIRVCapabilityKind &val) { return !CapMap.count(val); })); } if (AutoAddExtensions) { // While we are reading existing SPIR-V we need to read it as-is and don't // add required extensions for each entry automatically auto Ext = Entry->getRequiredExtension(); if (Ext.hasValue()) addExtension(Ext.getValue()); } return Entry; } bool SPIRVModuleImpl::exist(SPIRVId Id) const { return exist(Id, nullptr); } bool SPIRVModuleImpl::exist(SPIRVId Id, SPIRVEntry **Entry) const { assert(Id != SPIRVID_INVALID && "Invalid Id"); SPIRVIdToEntryMap::const_iterator Loc = IdEntryMap.find(Id); if (Loc == IdEntryMap.end()) return false; if (Entry) *Entry = Loc->second; return true; } // If Id is invalid, returns the next available id. // Otherwise returns the given id and adjust the next available id by increment. SPIRVId SPIRVModuleImpl::getId(SPIRVId Id, unsigned Increment) { if (!isValidId(Id)) Id = NextId; else NextId = std::max(Id, NextId); NextId += Increment; return Id; } SPIRVEntry *SPIRVModuleImpl::getEntry(SPIRVId Id) const { assert(Id != SPIRVID_INVALID && "Invalid Id"); SPIRVIdToEntryMap::const_iterator Loc = IdEntryMap.find(Id); if (Loc != IdEntryMap.end()) { return Loc->second; } SPIRVIdToEntryMap::const_iterator LocFwd = IdTypeForwardMap.find(Id); if (LocFwd != IdTypeForwardMap.end()) { return LocFwd->second; } assert(false && "Id is not in map"); return nullptr; } SPIRVExtInstSetKind SPIRVModuleImpl::getBuiltinSet(SPIRVId SetId) const { auto Loc = IdToInstSetMap.find(SetId); assert(Loc != IdToInstSetMap.end() && "Invalid builtin set id"); return Loc->second; } bool SPIRVModuleImpl::isEntryPoint(SPIRVExecutionModelKind ExecModel, SPIRVId EP) const { assert(isValid(ExecModel) && "Invalid execution model"); assert(EP != SPIRVID_INVALID && "Invalid function id"); auto Loc = EntryPointSet.find(ExecModel); if (Loc == EntryPointSet.end()) { Loc = ConditionalEntryPointSet.find(ExecModel); if (Loc == ConditionalEntryPointSet.end()) return false; } return Loc->second.count(EP); } // Module change functions bool SPIRVModuleImpl::importBuiltinSet(const std::string &BuiltinSetName, SPIRVId *BuiltinSetId) { SPIRVId TmpBuiltinSetId = getId(); if (!importBuiltinSetWithId(BuiltinSetName, TmpBuiltinSetId)) return false; if (BuiltinSetId) *BuiltinSetId = TmpBuiltinSetId; return true; } bool SPIRVModuleImpl::importBuiltinSetWithId(const std::string &BuiltinSetName, SPIRVId BuiltinSetId) { SPIRVExtInstSetKind BuiltinSet = SPIRVEIS_Count; SPIRVCKRT(SPIRVBuiltinSetNameMap::rfind(BuiltinSetName, &BuiltinSet), InvalidBuiltinSetName, "Actual is " + BuiltinSetName); IdToInstSetMap[BuiltinSetId] = BuiltinSet; ExtInstSetIds[BuiltinSet] = BuiltinSetId; return true; } void SPIRVModuleImpl::setAlignment(SPIRVValue *V, SPIRVWord A) { V->setAlignment(A); } void SPIRVModuleImpl::setName(SPIRVEntry *E, const std::string &Name) { E->setName(Name); if (!E->hasId()) return; if (!Name.empty()) NamedId.insert(E->getId()); else NamedId.erase(E->getId()); } void SPIRVModuleImpl::resolveUnknownStructFields() { for (auto &KV : UnknownStructFieldMap) { auto *Struct = KV.first; for (auto &Indices : KV.second) { unsigned I = Indices.first; SPIRVId ID = Indices.second; auto Ty = static_cast(getEntry(ID)); Struct->setMemberType(I, Ty); } } } // Type creation functions template T *SPIRVModuleImpl::addType(T *Ty) { add(Ty); if (!Ty->getName().empty()) setName(Ty, Ty->getName()); return Ty; } SPIRVTypeVoid *SPIRVModuleImpl::addVoidType() { return addType(new SPIRVTypeVoid(this, getId())); } SPIRVTypeArray *SPIRVModuleImpl::addArrayType(SPIRVType *ElementType, SPIRVConstant *Length) { return addType(new SPIRVTypeArray(this, getId(), ElementType, Length)); } SPIRVTypeBool *SPIRVModuleImpl::addBoolType() { return addType(new SPIRVTypeBool(this, getId())); } SPIRVTypeInt *SPIRVModuleImpl::addIntegerType(unsigned BitWidth) { auto Loc = IntTypeMap.find(BitWidth); if (Loc != IntTypeMap.end()) return Loc->second; auto Ty = new SPIRVTypeInt(this, getId(), BitWidth, false); IntTypeMap[BitWidth] = Ty; return addType(Ty); } SPIRVTypeFloat *SPIRVModuleImpl::addFloatType(unsigned BitWidth, unsigned FloatingPointEncoding) { auto Desc = std::make_pair(BitWidth, FloatingPointEncoding); auto Loc = FloatTypeMap.find(Desc); if (Loc != FloatTypeMap.end()) return Loc->second; auto *Ty = new SPIRVTypeFloat(this, getId(), BitWidth, FloatingPointEncoding); FloatTypeMap[Desc] = Ty; return addType(Ty); } SPIRVTypePointer * SPIRVModuleImpl::addPointerType(SPIRVStorageClassKind StorageClass, SPIRVType *ElementType) { return addType( new SPIRVTypePointer(this, getId(), StorageClass, ElementType)); } SPIRVTypeFunction *SPIRVModuleImpl::addFunctionType( SPIRVType *ReturnType, const std::vector &ParameterTypes) { return addType( new SPIRVTypeFunction(this, getId(), ReturnType, ParameterTypes)); } SPIRVTypeOpaque *SPIRVModuleImpl::addOpaqueType(const std::string &Name) { return addType(new SPIRVTypeOpaque(this, getId(), Name)); } SPIRVTypeStruct *SPIRVModuleImpl::openStructType(unsigned NumMembers, const std::string &Name) { auto T = new SPIRVTypeStruct(this, getId(), NumMembers, Name); return T; } SPIRVEntry *SPIRVModuleImpl::addTypeStructContinuedINTEL(unsigned NumMembers) { return add(new SPIRVTypeStructContinuedINTEL(this, NumMembers)); } void SPIRVModuleImpl::closeStructType(SPIRVTypeStruct *T, bool Packed) { addType(T); T->setPacked(Packed); } SPIRVTypeVector *SPIRVModuleImpl::addVectorType(SPIRVType *CompType, SPIRVWord CompCount) { return addType(new SPIRVTypeVector(this, getId(), CompType, CompCount)); } SPIRVTypeJointMatrixINTEL * SPIRVModuleImpl::addJointMatrixINTELType(SPIRVType *CompType, std::vector Args) { return addType(new SPIRVTypeJointMatrixINTEL(this, getId(), CompType, Args)); } SPIRVTypeCooperativeMatrixKHR * SPIRVModuleImpl::addCooperativeMatrixKHRType(SPIRVType *CompType, std::vector Args) { return addType( new SPIRVTypeCooperativeMatrixKHR(this, getId(), CompType, Args)); } SPIRVInstruction *SPIRVModuleImpl::addCooperativeMatrixLengthKHRInst( SPIRVType *RetTy, SPIRVType *MatTy, SPIRVBasicBlock *BB) { return addInstruction( SPIRVInstTemplateBase::create(OpCooperativeMatrixLengthKHR, RetTy, getId(), getVec(MatTy->getId()), BB, this), BB); } SPIRVType *SPIRVModuleImpl::addOpaqueGenericType(Op TheOpCode) { return addType(new SPIRVTypeOpaqueGeneric(TheOpCode, this, getId())); } SPIRVTypeDeviceEvent *SPIRVModuleImpl::addDeviceEventType() { return addType(new SPIRVTypeDeviceEvent(this, getId())); } SPIRVTypeQueue *SPIRVModuleImpl::addQueueType() { return addType(new SPIRVTypeQueue(this, getId())); } SPIRVTypePipe *SPIRVModuleImpl::addPipeType() { return addType(new SPIRVTypePipe(this, getId())); } SPIRVTypeImage * SPIRVModuleImpl::addImageType(SPIRVType *SampledType, const SPIRVTypeImageDescriptor &Desc) { return addType(new SPIRVTypeImage( this, getId(), SampledType ? SampledType->getId() : 0, Desc)); } SPIRVTypeImage * SPIRVModuleImpl::addImageType(SPIRVType *SampledType, const SPIRVTypeImageDescriptor &Desc, SPIRVAccessQualifierKind Acc) { return addType(new SPIRVTypeImage( this, getId(), SampledType ? SampledType->getId() : 0, Desc, Acc)); } SPIRVTypeSampler *SPIRVModuleImpl::addSamplerType() { return addType(new SPIRVTypeSampler(this, getId())); } SPIRVTypePipeStorage *SPIRVModuleImpl::addPipeStorageType() { return addType(new SPIRVTypePipeStorage(this, getId())); } SPIRVTypeSampledImage *SPIRVModuleImpl::addSampledImageType(SPIRVTypeImage *T) { return addType(new SPIRVTypeSampledImage(this, getId(), T)); } SPIRVTypeVmeImageINTEL * SPIRVModuleImpl::addVmeImageINTELType(SPIRVTypeImage *T) { return addType(new SPIRVTypeVmeImageINTEL(this, getId(), T)); } SPIRVTypeBufferSurfaceINTEL * SPIRVModuleImpl::addBufferSurfaceINTELType(SPIRVAccessQualifierKind Access) { return addType(new SPIRVTypeBufferSurfaceINTEL(this, getId(), Access)); } SPIRVType *SPIRVModuleImpl::addSubgroupAvcINTELType(Op TheOpCode) { return addType(new SPIRVTypeSubgroupAvcINTEL(TheOpCode, this, getId())); } SPIRVTypeTokenINTEL *SPIRVModuleImpl::addTokenTypeINTEL() { return addType(new SPIRVTypeTokenINTEL(this, getId())); } SPIRVFunction *SPIRVModuleImpl::addFunction(SPIRVFunction *Func) { FuncVec.push_back(add(Func)); return Func; } SPIRVFunction *SPIRVModuleImpl::addFunction(SPIRVTypeFunction *FuncType, SPIRVId Id) { return addFunction(new SPIRVFunction( this, FuncType, getId(Id, FuncType->getNumParameters() + 1))); } SPIRVBasicBlock *SPIRVModuleImpl::addBasicBlock(SPIRVFunction *Func, SPIRVId Id) { return Func->addBasicBlock(new SPIRVBasicBlock(getId(Id), Func)); } const SPIRVDecorateGeneric * SPIRVModuleImpl::addDecorate(SPIRVDecorateGeneric *Dec) { add(Dec); SPIRVId Id = Dec->getTargetId(); bool Found = exist(Id); (void)Found; assert(Found && "Decorate target does not exist"); if (!Dec->getOwner()) DecorateVec.push_back(Dec); addCapabilities(Dec->getRequiredCapability()); return Dec; } void SPIRVModuleImpl::addEntryPoint(SPIRVExecutionModelKind ExecModel, SPIRVId EntryPoint, const std::string &Name, const std::vector &Variables) { assert(isValid(ExecModel) && "Invalid execution model"); assert(EntryPoint != SPIRVID_INVALID && "Invalid entry point"); auto *EP = add(new SPIRVEntryPoint(this, ExecModel, EntryPoint, Name, Variables)); EntryPointVec.push_back(EP); EntryPointSet[ExecModel].insert(EntryPoint); addCapabilities(SPIRV::getCapability(ExecModel)); } void SPIRVModuleImpl::addConditionalEntryPoint( SPIRVId Condition, SPIRVExecutionModelKind ExecModel, SPIRVId EntryPoint, const std::string &Name, const std::vector &Variables) { assert(isValid(ExecModel) && "Invalid execution model"); assert(EntryPoint != SPIRVID_INVALID && "Invalid entry point"); auto *EP = add(new SPIRVConditionalEntryPointINTEL( this, Condition, ExecModel, EntryPoint, Name, Variables)); ConditionalEntryPointVec.push_back(EP); ConditionalEntryPointSet[ExecModel].insert(EntryPoint); } void SPIRVModuleImpl::specializeConditionalEntryPoints(SPIRVId Condition, bool ShouldKeep) { std::vector EPsToRemove; std::vector EPIdsToRemove; for (const auto *EP : ConditionalEntryPointVec) { if (EP->getCondition() == Condition) { EPsToRemove.push_back(EP); EPIdsToRemove.push_back(EP->getTargetId()); if (ShouldKeep) { // add the removed conditional entry point as a normal entry point addEntryPoint(EP->getExecModel(), EP->getTargetId(), EP->getName(), EP->getVariables()); } } } erase_if(ConditionalEntryPointVec, [&EPsToRemove](const auto *EP) { return std::find(EPsToRemove.begin(), EPsToRemove.end(), EP) != EPsToRemove.end(); }); for (const auto &Id : EPIdsToRemove) { for (auto &It : ConditionalEntryPointSet) { It.second.erase(Id); } } } SPIRVForward *SPIRVModuleImpl::addForward(SPIRVType *Ty) { return add(new SPIRVForward(this, Ty, getId())); } SPIRVForward *SPIRVModuleImpl::addForward(SPIRVId Id, SPIRVType *Ty) { return add(new SPIRVForward(this, Ty, Id)); } SPIRVEntry *SPIRVModuleImpl::replaceForward(SPIRVForward *Forward, SPIRVEntry *Entry) { SPIRVId Id = Entry->getId(); SPIRVId ForwardId = Forward->getId(); if (ForwardId == Id) { IdEntryMap[Id] = Entry; // Annotations include name, decorations, execution modes Entry->takeAnnotations(Forward); } else { auto Loc = IdEntryMap.find(Id); assert(Loc != IdEntryMap.end()); IdEntryMap.erase(Loc); Entry->setId(ForwardId); IdEntryMap[ForwardId] = Entry; // Replace current Id with ForwardId in decorates. Entry->replaceTargetIdInDecorates(ForwardId); } delete Forward; return Entry; } void SPIRVModuleImpl::eraseInstruction(SPIRVInstruction *I, SPIRVBasicBlock *BB) { SPIRVId Id = I->getId(); BB->eraseInstruction(I); auto Loc = IdEntryMap.find(Id); assert(Loc != IdEntryMap.end()); IdEntryMap.erase(Loc); delete I; } bool SPIRVModuleImpl::eraseValue(SPIRVValue *V) { const Op OpCode = V->getOpCode(); SPIRVId Id = V->getId(); if (isTypeOpCode(OpCode)) { erase_if(TypeVec, [Id](auto T) { return T->getId() == Id; }); } else if (OpCode == OpVariable) { erase_if(VariableVec, [Id](auto V) { return V->getId() == Id; }); } else if (isConstantOpCode(OpCode)) { erase_if(ConstVec, [Id](auto C) { return C->getId() == Id; }); } else if (OpCode == Op::OpAsmINTEL) { erase_if(AsmVec, [Id](auto A) { return A->getId() == Id; }); } else if (OpCode == Op::OpAsmTargetINTEL) { erase_if(AsmTargetVec, [Id](auto AT) { return AT->getId() == Id; }); } else { return false; } auto Loc = IdEntryMap.find(Id); assert(Loc != IdEntryMap.end()); IdEntryMap.erase(Loc); delete V; return true; } SPIRVValue *SPIRVModuleImpl::addConstant(SPIRVValue *C) { return add(C); } SPIRVValue *SPIRVModuleImpl::addConstant(SPIRVType *Ty, uint64_t V) { if (Ty->isTypeBool()) { if (V) return addConstant(new SPIRVConstantTrue(this, Ty, getId())); else return addConstant(new SPIRVConstantFalse(this, Ty, getId())); } if (Ty->isTypeInt()) return addIntegerConstant(static_cast(Ty), V); return addConstant(new SPIRVConstant(this, Ty, getId(), V)); } SPIRVValue *SPIRVModuleImpl::addConstant(SPIRVType *Ty, llvm::APInt V) { return addConstant(new SPIRVConstant(this, Ty, getId(), V)); } SPIRVValue *SPIRVModuleImpl::addIntegerConstant(SPIRVTypeInt *Ty, uint64_t V) { if (Ty->getBitWidth() == 32) { unsigned I32 = static_cast(V); assert(I32 == V && "Integer value truncated"); return getLiteralAsConstant(I32); } return addConstant(new SPIRVConstant(this, Ty, getId(), V)); } SPIRVValue *SPIRVModuleImpl::addFloatConstant(SPIRVTypeFloat *Ty, float V) { return addConstant(new SPIRVConstant(this, Ty, getId(), V)); } SPIRVValue *SPIRVModuleImpl::addDoubleConstant(SPIRVTypeFloat *Ty, double V) { return addConstant(new SPIRVConstant(this, Ty, getId(), V)); } SPIRVValue *SPIRVModuleImpl::addNullConstant(SPIRVType *Ty) { return addConstant(new SPIRVConstantNull(this, Ty, getId())); } SPIRVValue *SPIRVModuleImpl::addCompositeConstant( SPIRVType *Ty, const std::vector &Elements) { constexpr int MaxNumElements = MaxWordCount - SPIRVConstantComposite::FixedWC; const int NumElements = Elements.size(); // In case number of elements is greater than maximum WordCount and // SPV_INTEL_long_constant_composite is not enabled, the error will be emitted // by validate functionality of SPIRVCompositeConstant class. if (NumElements <= MaxNumElements || !isAllowedToUseExtension(ExtensionID::SPV_INTEL_long_constant_composite)) return addConstant(new SPIRVConstantComposite(this, Ty, getId(), Elements)); auto Start = Elements.begin(); auto End = Start + MaxNumElements; std::vector Slice(Start, End); auto *Res = static_cast(addCompositeConstant(Ty, Slice)); for (; End != Elements.end();) { Start = End; End = ((Elements.end() - End) > MaxNumElements) ? End + MaxNumElements : Elements.end(); Slice.assign(Start, End); auto Continued = static_cast( addCompositeConstantContinuedINTEL(Slice)); Res->addContinuedInstruction(Continued); } return Res; } SPIRVEntry *SPIRVModuleImpl::addCompositeConstantContinuedINTEL( const std::vector &Elements) { return add(new SPIRVConstantCompositeContinuedINTEL(this, Elements)); } SPIRVValue *SPIRVModuleImpl::addSpecConstantComposite( SPIRVType *Ty, const std::vector &Elements) { constexpr int MaxNumElements = MaxWordCount - SPIRVSpecConstantComposite::FixedWC; const int NumElements = Elements.size(); // In case number of elements is greater than maximum WordCount and // SPV_INTEL_long_constant_composite is not enabled, the error will be emitted // by validate functionality of SPIRVSpecConstantComposite class. if (NumElements <= MaxNumElements || !isAllowedToUseExtension(ExtensionID::SPV_INTEL_long_constant_composite)) return addConstant( new SPIRVSpecConstantComposite(this, Ty, getId(), Elements)); auto Start = Elements.begin(); auto End = Start + MaxNumElements; std::vector Slice(Start, End); auto *Res = static_cast( addSpecConstantComposite(Ty, Slice)); for (; End != Elements.end();) { Start = End; End = ((Elements.end() - End) > MaxNumElements) ? End + MaxNumElements : Elements.end(); Slice.assign(Start, End); auto Continued = static_cast( addSpecConstantCompositeContinuedINTEL(Slice)); Res->addContinuedInstruction(Continued); } return Res; } SPIRVEntry *SPIRVModuleImpl::addSpecConstantCompositeContinuedINTEL( const std::vector &Elements) { return add(new SPIRVSpecConstantCompositeContinuedINTEL(this, Elements)); } SPIRVValue *SPIRVModuleImpl::addConstantFunctionPointerINTEL(SPIRVType *Ty, SPIRVFunction *F) { return addConstant( new SPIRVConstantFunctionPointerINTEL(getId(), Ty, F, this)); } SPIRVValue *SPIRVModuleImpl::addUndef(SPIRVType *TheType) { return addConstant(new SPIRVUndef(this, TheType, getId())); } SPIRVValue *SPIRVModuleImpl::addSpecConstant(SPIRVType *Ty, uint64_t V) { if (Ty->isTypeBool()) { if (V) return add(new SPIRVSpecConstantTrue(this, Ty, getId())); else return add(new SPIRVSpecConstantFalse(this, Ty, getId())); } return add(new SPIRVSpecConstant(this, Ty, getId(), V)); } // Instruction creation functions SPIRVInstruction * SPIRVModuleImpl::addStoreInst(SPIRVValue *Target, SPIRVValue *Source, const std::vector &TheMemoryAccess, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVStore(Target->getId(), Source->getId(), TheMemoryAccess, BB)); } SPIRVInstruction *SPIRVModuleImpl::addSwitchInst( SPIRVValue *Select, SPIRVBasicBlock *Default, const std::vector, SPIRVBasicBlock *>> &Pairs, SPIRVBasicBlock *BB) { return BB->addInstruction(new SPIRVSwitch(Select, Default, Pairs, BB)); } SPIRVInstruction * SPIRVModuleImpl::addVectorTimesScalarInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheScalar, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVVectorTimesScalar(TheType, getId(), TheVector, TheScalar, BB)); } SPIRVInstruction * SPIRVModuleImpl::addVectorTimesMatrixInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheMatrix, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVVectorTimesMatrix(TheType, getId(), TheVector, TheMatrix, BB)); } SPIRVInstruction * SPIRVModuleImpl::addMatrixTimesScalarInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheScalar, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVMatrixTimesScalar(TheType, getId(), TheMatrix, TheScalar, BB)); } SPIRVInstruction * SPIRVModuleImpl::addMatrixTimesVectorInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheVector, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVMatrixTimesVector(TheType, getId(), TheMatrix, TheVector, BB)); } SPIRVInstruction * SPIRVModuleImpl::addMatrixTimesMatrixInst(SPIRVType *TheType, SPIRVId M1, SPIRVId M2, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVMatrixTimesMatrix(TheType, getId(), M1, M2, BB)); } SPIRVInstruction *SPIRVModuleImpl::addTransposeInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVBasicBlock *BB) { return BB->addInstruction( new SPIRVTranspose(TheType, getId(), TheMatrix, BB)); } SPIRVInstruction * SPIRVModuleImpl::addGroupInst(Op OpCode, SPIRVType *Type, Scope Scope, const std::vector &Ops, SPIRVBasicBlock *BB) { assert(!Type || !Type->isTypeVoid()); auto WordOps = getIds(Ops); WordOps.insert(WordOps.begin(), Scope); return addInstTemplate(OpCode, WordOps, BB, Type); } SPIRVInstruction * SPIRVModuleImpl::addInstruction(SPIRVInstruction *Inst, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore) { if (BB) return BB->addInstruction(Inst, InsertBefore); if (Inst->getOpCode() != OpSpecConstantOp) { SPIRVInstruction *Res = createSpecConstantOpInst(Inst); delete Inst; Inst = Res; } return static_cast(addConstant(Inst)); } SPIRVInstruction * SPIRVModuleImpl::addLoadInst(SPIRVValue *Source, const std::vector &TheMemoryAccess, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVLoad(getId(), Source->getId(), TheMemoryAccess, BB), BB); } SPIRVInstruction * SPIRVModuleImpl::addPhiInst(SPIRVType *Type, std::vector IncomingPairs, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVPhi(Type, getId(), IncomingPairs, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addExtInst( SPIRVType *TheType, SPIRVWord BuiltinSet, SPIRVWord EntryPoint, const std::vector &Args, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore) { return addInstruction( new SPIRVExtInst(TheType, getId(), BuiltinSet, EntryPoint, Args, BB), BB, InsertBefore); } SPIRVInstruction *SPIRVModuleImpl::addExtInst( SPIRVType *TheType, SPIRVWord BuiltinSet, SPIRVWord EntryPoint, const std::vector &Args, SPIRVBasicBlock *BB, SPIRVInstruction *InsertBefore) { return addInstruction( new SPIRVExtInst(TheType, getId(), BuiltinSet, EntryPoint, Args, BB), BB, InsertBefore); } SPIRVEntry * SPIRVModuleImpl::createDebugInfo(SPIRVWord InstId, SPIRVType *TheType, const std::vector &Args) { return new SPIRVExtInst(this, getId(), TheType, SPIRVEIS_OpenCL_DebugInfo_100, ExtInstSetIds[getDebugInfoEIS()], InstId, Args); } SPIRVEntry *SPIRVModuleImpl::addDebugInfo(SPIRVWord InstId, SPIRVType *TheType, const std::vector &Args) { return addEntry(createDebugInfo(InstId, TheType, Args)); } SPIRVEntry *SPIRVModuleImpl::addAuxData(SPIRVWord InstId, SPIRVType *TheType, const std::vector &Args) { return addEntry(new SPIRVExtInst( this, getId(), TheType, SPIRVEIS_NonSemantic_AuxData, getExtInstSetId(SPIRVEIS_NonSemantic_AuxData), InstId, Args)); } SPIRVEntry *SPIRVModuleImpl::addModuleProcessed(const std::string &Process) { ModuleProcessedVec.push_back(new SPIRVModuleProcessed(this, Process)); return ModuleProcessedVec.back(); } std::vector SPIRVModuleImpl::getModuleProcessedVec() { return ModuleProcessedVec; } SPIRVInstruction * SPIRVModuleImpl::addCallInst(SPIRVFunction *TheFunction, const std::vector &TheArguments, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVFunctionCall(getId(), TheFunction, TheArguments, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addIndirectCallInst( SPIRVValue *TheCalledValue, SPIRVType *TheReturnType, const std::vector &TheArguments, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVFunctionPointerCallINTEL(getId(), TheCalledValue, TheReturnType, TheArguments, BB), BB); } SPIRVEntry * SPIRVModuleImpl::getOrAddAsmTargetINTEL(const std::string &TheTarget) { auto TargetIt = std::find_if(AsmTargetVec.begin(), AsmTargetVec.end(), [&TheTarget](const SPIRVAsmTargetINTEL *Target) { return Target->getTarget() == TheTarget; }); if (TargetIt == AsmTargetVec.end()) return add(new SPIRVAsmTargetINTEL(this, getId(), TheTarget)); return *TargetIt; } SPIRVValue *SPIRVModuleImpl::addAsmINTEL(SPIRVTypeFunction *TheType, SPIRVAsmTargetINTEL *TheTarget, const std::string &TheInstructions, const std::string &TheConstraints) { auto Asm = new SPIRVAsmINTEL(this, TheType, getId(), TheTarget, TheInstructions, TheConstraints); return add(Asm); } SPIRVInstruction * SPIRVModuleImpl::addAsmCallINTELInst(SPIRVAsmINTEL *TheAsm, const std::vector &TheArguments, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVAsmCallINTEL(getId(), TheAsm, TheArguments, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addBinaryInst(Op TheOpCode, SPIRVType *Type, SPIRVValue *Op1, SPIRVValue *Op2, SPIRVBasicBlock *BB) { if (Type->isTypeFloat(16, FPEncodingBFloat16KHR) && TheOpCode != OpDot) addCapability(internal::CapabilityBFloat16ArithmeticINTEL); return addInstruction(SPIRVInstTemplateBase::create( TheOpCode, Type, getId(), getVec(Op1->getId(), Op2->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addUnreachableInst(SPIRVBasicBlock *BB) { return addInstruction(new SPIRVUnreachable(BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addReturnInst(SPIRVBasicBlock *BB) { return addInstruction(new SPIRVReturn(BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addReturnValueInst(SPIRVValue *ReturnValue, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVReturnValue(ReturnValue, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addUnaryInst(Op TheOpCode, SPIRVType *TheType, SPIRVValue *Op, SPIRVBasicBlock *BB) { if (TheType->isTypeFloat(16, FPEncodingBFloat16KHR) && TheOpCode != OpDot) addCapability(internal::CapabilityBFloat16ArithmeticINTEL); return addInstruction( SPIRVInstTemplateBase::create(TheOpCode, TheType, getId(), getVec(Op->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addVectorExtractDynamicInst( SPIRVValue *TheVector, SPIRVValue *Index, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVVectorExtractDynamic(getId(), TheVector, Index, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addVectorInsertDynamicInst( SPIRVValue *TheVector, SPIRVValue *TheComponent, SPIRVValue *Index, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVVectorInsertDynamic(getId(), TheVector, TheComponent, Index, BB), BB); } SPIRVValue *SPIRVModuleImpl::addVectorShuffleInst( SPIRVType *Type, SPIRVValue *Vec1, SPIRVValue *Vec2, const std::vector &Components, SPIRVBasicBlock *BB) { std::vector Ops{Vec1->getId(), Vec2->getId()}; Ops.insert(Ops.end(), Components.begin(), Components.end()); return addInstruction(SPIRVInstTemplateBase::create(OpVectorShuffle, Type, getId(), Ops, BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addBranchInst(SPIRVLabel *TargetLabel, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVBranch(TargetLabel, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addBranchConditionalInst( SPIRVValue *Condition, SPIRVLabel *TrueLabel, SPIRVLabel *FalseLabel, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVBranchConditional(Condition, TrueLabel, FalseLabel, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addCmpInst(Op TheOpCode, SPIRVType *TheType, SPIRVValue *Op1, SPIRVValue *Op2, SPIRVBasicBlock *BB) { return addInstruction(SPIRVInstTemplateBase::create( TheOpCode, TheType, getId(), getVec(Op1->getId(), Op2->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addControlBarrierInst(SPIRVValue *ExecKind, SPIRVValue *MemKind, SPIRVValue *MemSema, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVControlBarrier(ExecKind, MemKind, MemSema, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addLifetimeInst(Op OC, SPIRVValue *Object, SPIRVWord Size, SPIRVBasicBlock *BB) { if (OC == OpLifetimeStart) return BB->addInstruction( new SPIRVLifetimeStart(Object->getId(), Size, BB)); else return BB->addInstruction(new SPIRVLifetimeStop(Object->getId(), Size, BB)); } SPIRVInstruction *SPIRVModuleImpl::addMemoryBarrierInst(Scope ScopeKind, SPIRVWord MemFlag, SPIRVBasicBlock *BB) { return addInstruction(SPIRVInstTemplateBase::create( OpMemoryBarrier, nullptr, SPIRVID_INVALID, getVec(static_cast(ScopeKind), MemFlag), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addSelectInst(SPIRVValue *Condition, SPIRVValue *Op1, SPIRVValue *Op2, SPIRVBasicBlock *BB) { return addInstruction( SPIRVInstTemplateBase::create( OpSelect, Op1->getType(), getId(), getVec(Condition->getId(), Op1->getId(), Op2->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addSelectionMergeInst( SPIRVId MergeBlock, SPIRVWord SelectionControl, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVSelectionMerge(MergeBlock, SelectionControl, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addLoopMergeInst( SPIRVId MergeBlock, SPIRVId ContinueTarget, SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVLoopMerge(MergeBlock, ContinueTarget, LoopControl, LoopControlParameters, BB), BB, const_cast(BB->getTerminateInstr())); } SPIRVInstruction *SPIRVModuleImpl::addLoopControlINTELInst( SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) { addCapability(CapabilityUnstructuredLoopControlsINTEL); addExtension(ExtensionID::SPV_INTEL_unstructured_loop_controls); return addInstruction( new SPIRVLoopControlINTEL(LoopControl, LoopControlParameters, BB), BB, const_cast(BB->getTerminateInstr())); } SPIRVInstruction *SPIRVModuleImpl::addFixedPointIntelInst( Op OC, SPIRVType *ResTy, SPIRVValue *Input, const std::vector &Ops, SPIRVBasicBlock *BB) { std::vector TheOps = getVec(Input->getId(), Ops); return addInstruction( SPIRVInstTemplateBase::create(OC, ResTy, getId(), TheOps, BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addArbFloatPointIntelInst( Op OC, SPIRVType *ResTy, SPIRVValue *InA, SPIRVValue *InB, const std::vector &Ops, SPIRVBasicBlock *BB) { // SPIR-V format: // A [Literal MA] [B] [Literal MB] [Literal Mout] [Literal Sign] // [Literal EnableSubnormals Literal RoundingMode Literal RoundingAccuracy] auto OpsItr = Ops.begin(); std::vector TheOps = getVec(InA->getId(), *OpsItr++); if (InB) TheOps.push_back(InB->getId()); TheOps.insert(TheOps.end(), OpsItr, Ops.end()); return addInstruction( SPIRVInstTemplateBase::create(OC, ResTy, getId(), TheOps, BB, this), BB); } SPIRVInstruction * SPIRVModuleImpl::addPtrAccessChainInst(SPIRVType *Type, SPIRVValue *Base, std::vector Indices, SPIRVBasicBlock *BB, bool IsInBounds) { return addInstruction( SPIRVInstTemplateBase::create( IsInBounds ? OpInBoundsPtrAccessChain : OpPtrAccessChain, Type, getId(), getVec(Base->getId(), Base->getIds(Indices)), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addAsyncGroupCopy( SPIRVValue *Scope, SPIRVValue *Dest, SPIRVValue *Src, SPIRVValue *NumElems, SPIRVValue *Stride, SPIRVValue *Event, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVGroupAsyncCopy(Scope, getId(), Dest, Src, NumElems, Stride, Event, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addCompositeConstructInst( SPIRVType *Type, const std::vector &Constituents, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVCompositeConstruct(Type, getId(), Constituents, BB), BB); } SPIRVInstruction * SPIRVModuleImpl::addCompositeExtractInst(SPIRVType *Type, SPIRVValue *TheVector, const std::vector &Indices, SPIRVBasicBlock *BB) { return addInstruction(SPIRVInstTemplateBase::create( OpCompositeExtract, Type, getId(), getVec(TheVector->getId(), Indices), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addCompositeInsertInst( SPIRVValue *Object, SPIRVValue *Composite, const std::vector &Indices, SPIRVBasicBlock *BB) { std::vector Ops{Object->getId(), Composite->getId()}; Ops.insert(Ops.end(), Indices.begin(), Indices.end()); return addInstruction(SPIRVInstTemplateBase::create(OpCompositeInsert, Composite->getType(), getId(), Ops, BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addCopyObjectInst(SPIRVType *TheType, SPIRVValue *Operand, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVCopyObject(TheType, getId(), Operand, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addCopyMemoryInst( SPIRVValue *TheTarget, SPIRVValue *TheSource, const std::vector &TheMemoryAccess, SPIRVBasicBlock *BB) { return addInstruction( new SPIRVCopyMemory(TheTarget, TheSource, TheMemoryAccess, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addCopyMemorySizedInst( SPIRVValue *TheTarget, SPIRVValue *TheSource, SPIRVValue *TheSize, const std::vector &TheMemoryAccess, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVCopyMemorySized(TheTarget, TheSource, TheSize, TheMemoryAccess, BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addFPGARegINTELInst(SPIRVType *Type, SPIRVValue *V, SPIRVBasicBlock *BB) { return addInstruction( SPIRVInstTemplateBase::create(OpFPGARegINTEL, Type, getId(), getVec(V->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addSampledImageInst(SPIRVType *ResultTy, SPIRVValue *Image, SPIRVValue *Sampler, SPIRVBasicBlock *BB) { return addInstruction(SPIRVInstTemplateBase::create( OpSampledImage, ResultTy, getId(), getVec(Image->getId(), Sampler->getId()), BB, this), BB); } SPIRVInstruction *SPIRVModuleImpl::addAssumeTrueKHRInst(SPIRVValue *Condition, SPIRVBasicBlock *BB) { return addInstruction(new SPIRVAssumeTrueKHR(Condition->getId(), BB), BB); } SPIRVInstruction *SPIRVModuleImpl::addExpectKHRInst(SPIRVType *ResultTy, SPIRVValue *Value, SPIRVValue *ExpectedValue, SPIRVBasicBlock *BB) { return addInstruction(SPIRVInstTemplateBase::create( OpExpectKHR, ResultTy, getId(), getVec(Value->getId(), ExpectedValue->getId()), BB, this), BB); } // Create AliasDomainDeclINTEL/AliasScopeDeclINTEL/AliasScopeListDeclINTEL // instructions template SPIRVEntry * SPIRVModuleImpl::getOrAddMemAliasingINTELInst(std::vector Args, llvm::MDNode *MD) { assert(MD && "noalias/alias.scope metadata can't be null"); // Don't duplicate aliasing instruction. For that use a map with a MDNode key if (AliasInstMDMap.find(MD) != AliasInstMDMap.end()) return AliasInstMDMap[MD]; SPIRVEntry *AliasInst = add(new AliasingInstType(this, getId(), Args)); AliasInstMDMap.emplace(std::make_pair(MD, AliasInst)); return AliasInst; } // Create AliasDomainDeclINTEL instruction SPIRVEntry * SPIRVModuleImpl::getOrAddAliasDomainDeclINTELInst(std::vector Args, llvm::MDNode *MD) { return getOrAddMemAliasingINTELInst(Args, MD); } // Create AliasScopeDeclINTEL instruction SPIRVEntry * SPIRVModuleImpl::getOrAddAliasScopeDeclINTELInst(std::vector Args, llvm::MDNode *MD) { return getOrAddMemAliasingINTELInst(Args, MD); } // Create AliasScopeListDeclINTEL instruction SPIRVEntry * SPIRVModuleImpl::getOrAddAliasScopeListDeclINTELInst(std::vector Args, llvm::MDNode *MD) { return getOrAddMemAliasingINTELInst(Args, MD); } SPIRVInstruction *SPIRVModuleImpl::addVariable( SPIRVType *Type, bool IsConstant, SPIRVLinkageTypeKind LinkageTy, SPIRVValue *Initializer, const std::string &Name, SPIRVStorageClassKind StorageClass, SPIRVBasicBlock *BB) { SPIRVVariable *Variable = new SPIRVVariable(Type, getId(), Initializer, Name, StorageClass, BB, this); if (BB) return addInstruction(Variable, BB); add(Variable); if (LinkageTy != internal::LinkageTypeInternal) Variable->setLinkageType(LinkageTy); Variable->setIsConstant(IsConstant); return Variable; } template spv_ostream &operator<<(spv_ostream &O, const std::vector &V) { for (auto &I : V) O << *I; return O; } template > spv_ostream &operator<<(spv_ostream &O, const std::unordered_set &V) { for (auto &I : V) O << *I; return O; } // To satisfy SPIR-V spec requirement: // "All operands must be declared before being used", // we do DFS based topological sort // https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search class TopologicalSort { enum DFSState : char { Unvisited, Discovered, Visited }; typedef std::vector SPIRVTypeVec; typedef std::vector SPIRVConstantVector; typedef std::vector SPIRVVariableVec; typedef std::vector SPIRVConstAndVarVec; typedef std::vector SPIRVForwardPointerVec; typedef std::function Comp; typedef std::map EntryStateMapTy; typedef std::function Equal; typedef std::function Hash; // We may create forward pointers as we go through the types. We use // unordered set to avoid duplicates. typedef std::unordered_set SPIRVForwardPointerSet; SPIRVTypeVec TypeIntVec; SPIRVConstantVector ConstIntVec; SPIRVTypeVec TypeVec; SPIRVConstAndVarVec ConstAndVarVec; SPIRVForwardPointerSet ForwardPointerSet; EntryStateMapTy EntryStateMap; friend spv_ostream &operator<<(spv_ostream &O, const TopologicalSort &S); // This method implements recursive depth-first search among all Entries in // EntryStateMap. Traversing entries and adding them to corresponding // container after visiting all dependent entries(post-order traversal) // guarantees that the entry's operands will appear in the container before // the entry itslef. // Returns true if cyclic dependency detected. bool visit(SPIRVEntry *E) { DFSState &State = EntryStateMap[E]; if (State == Visited) return false; if (State == Discovered) // Cyclic dependency detected return true; State = Discovered; for (SPIRVEntry *Op : E->getNonLiteralOperands()) { if (Op->getOpCode() == OpTypeForwardPointer) { SPIRVEntry *FP = E->getModule()->getEntry( static_cast(Op)->getPointerId()); Op = FP; } if (EntryStateMap[Op] == Visited) continue; if (visit(Op)) { // We've found a recursive data type, e.g. a structure having a member // which is a pointer to the same structure. State = Unvisited; // Forget about it if (E->getOpCode() == OpTypePointer) { // If we have a pointer in the recursive chain, we can break the // cyclic dependency by inserting a forward declaration of that // pointer. SPIRVTypePointer *Ptr = static_cast(E); SPIRVModule *BM = E->getModule(); ForwardPointerSet.insert(BM->add(new SPIRVTypeForwardPointer( BM, Ptr->getId(), Ptr->getPointerStorageClass()))); return false; } return true; } } Op OC = E->getOpCode(); if (OC == OpTypeInt) TypeIntVec.push_back(static_cast(E)); else if (isConstantOpCode(OC)) { SPIRVConstant *C = static_cast(E); if (C->getType()->isTypeInt()) ConstIntVec.push_back(C); else ConstAndVarVec.push_back(E); } else if (isTypeOpCode(OC)) TypeVec.push_back(static_cast(E)); else ConstAndVarVec.push_back(E); State = Visited; return false; } public: TopologicalSort(const SPIRVTypeVec &TypeVec, const SPIRVConstantVector &ConstVec, const SPIRVVariableVec &VariableVec, SPIRVForwardPointerVec &ForwardPointerVec) : ForwardPointerSet( 16, // bucket count [](const SPIRVTypeForwardPointer *Ptr) { return std::hash()(Ptr->getPointerId()); }, [](const SPIRVTypeForwardPointer *Ptr1, const SPIRVTypeForwardPointer *Ptr2) { return Ptr1->getPointerId() == Ptr2->getPointerId(); }), EntryStateMap([](SPIRVEntry *A, SPIRVEntry *B) -> bool { return A->getId() < B->getId(); }) { // Collect entries for sorting for (auto *T : TypeVec) EntryStateMap[T] = DFSState::Unvisited; for (auto *C : ConstVec) EntryStateMap[C] = DFSState::Unvisited; for (auto *V : VariableVec) EntryStateMap[V] = DFSState::Unvisited; // Run topoligical sort for (auto ES : EntryStateMap) { if (visit(ES.first)) llvm_unreachable("Cyclic dependency for types detected"); } // Append forward pointers vector ForwardPointerVec.insert(ForwardPointerVec.end(), ForwardPointerSet.begin(), ForwardPointerSet.end()); } }; spv_ostream &operator<<(spv_ostream &O, const TopologicalSort &S) { O << S.TypeIntVec << S.ConstIntVec << S.TypeVec << S.ConstAndVarVec; return O; } spv_ostream &operator<<(spv_ostream &O, SPIRVModule &M) { SPIRVModuleImpl &MI = *static_cast(&M); // Start tracking of the current line with no line MI.CurrentLine.reset(); MI.CurrentDebugLine.reset(); SPIRVEncoder Encoder(O); Encoder << MagicNumber << MI.SPIRVVersion << (((SPIRVWord)MI.GeneratorId << 16) | MI.GeneratorVer) << MI.NextId /* Bound for Id */ << MI.InstSchema; O << SPIRVNL(); for (auto &I : MI.CapMap) O << *I.second; for (auto &I : MI.ConditionalCapMap) O << *I.second; for (auto &I : M.getExtension()) { assert(!I.empty() && "Invalid extension"); O << SPIRVExtension(&M, I); } for (auto &I : M.getConditionalExtensions()) { auto Cond = I.first; auto Ext = I.second; assert(!Ext.empty() && "Invalid conditional extension"); O << SPIRVConditionalExtensionINTEL(&M, Cond, Ext); } for (auto &I : MI.IdToInstSetMap) O << SPIRVExtInstImport(&M, I.first, SPIRVBuiltinSetNameMap::map(I.second)); O << SPIRVMemoryModel(&M); O << MI.EntryPointVec; O << MI.ConditionalEntryPointVec; for (auto &I : MI.EntryPointVec) MI.get(I->getTargetId())->encodeExecutionModes(O); for (auto &I : MI.ConditionalEntryPointVec) MI.get(I->getTargetId())->encodeExecutionModes(O); O << MI.StringVec; for (auto &I : M.getSourceExtension()) { assert(!I.empty() && "Invalid source extension"); O << SPIRVSourceExtension(&M, I); } O << SPIRVSource(&M); for (auto &I : MI.NamedId) { // Don't output name for entry point since it is redundant bool IsEntryPoint = false; for (auto &EPS : MI.EntryPointSet) if (EPS.second.count(I)) { IsEntryPoint = true; break; } for (auto &EPS : MI.ConditionalEntryPointSet) if (EPS.second.count(I)) { IsEntryPoint = true; break; } if (!IsEntryPoint) M.getEntry(I)->encodeName(O); } if (M.isAllowedToUseExtension( ExtensionID::SPV_INTEL_memory_access_aliasing)) { O << SPIRVNL() << MI.AliasInstMDVec; } TopologicalSort TS(MI.TypeVec, MI.ConstVec, MI.VariableVec, MI.ForwardPointerVec); O << MI.MemberNameVec << MI.ModuleProcessedVec << MI.DecGroupVec << MI.DecorateVec << MI.GroupDecVec << MI.ForwardPointerVec << TS; if (M.isAllowedToUseExtension(ExtensionID::SPV_INTEL_inline_assembly)) { O << SPIRVNL() << MI.AsmTargetVec << MI.AsmVec; } // At this point we know that FunctionDefinition could have been included both // into DebugInstVec and into basick block of function from FuncVec. // By spec we should only have this instruction to be present inside the // function body, so removing it from the DebugInstVec to avoid duplication. MI.DebugInstVec.erase( std::remove_if(MI.DebugInstVec.begin(), MI.DebugInstVec.end(), [](SPIRVExtInst *I) { return I->getExtOp() == SPIRVDebug::FunctionDefinition; }), MI.DebugInstVec.end()); O << SPIRVNL() << MI.DebugInstVec << MI.AuxDataInstVec << SPIRVNL() << MI.FuncVec; return O; } template void SPIRVModuleImpl::addTo(std::vector &V, SPIRVEntry *E) { V.push_back(static_cast(E)); } // The first decoration group includes all the previously defined decorates. // The second decoration group includes all the decorates defined between the // first and second decoration group. So long so forth. SPIRVDecorationGroup *SPIRVModuleImpl::addDecorationGroup() { return addDecorationGroup(new SPIRVDecorationGroup(this, getId())); } SPIRVDecorationGroup * SPIRVModuleImpl::addDecorationGroup(SPIRVDecorationGroup *Group) { add(Group); Group->takeDecorates(DecorateVec); DecGroupVec.push_back(Group); SPIRVDBG(spvdbgs() << "[addDecorationGroup] {" << *Group << "}\n"; spvdbgs() << " Remaining DecorateVec: {" << DecorateVec << "}\n"); assert(DecorateVec.empty()); return Group; } SPIRVGroupDecorateGeneric * SPIRVModuleImpl::addGroupDecorateGeneric(SPIRVGroupDecorateGeneric *GDec) { add(GDec); GDec->decorateTargets(); GroupDecVec.push_back(GDec); return GDec; } SPIRVGroupDecorate * SPIRVModuleImpl::addGroupDecorate(SPIRVDecorationGroup *Group, const std::vector &Targets) { auto GD = new SPIRVGroupDecorate(Group, getIds(Targets)); addGroupDecorateGeneric(GD); return GD; } SPIRVGroupMemberDecorate *SPIRVModuleImpl::addGroupMemberDecorate( SPIRVDecorationGroup *Group, const std::vector &Targets) { auto GMD = new SPIRVGroupMemberDecorate(Group, getIds(Targets)); addGroupDecorateGeneric(GMD); return GMD; } SPIRVString *SPIRVModuleImpl::getString(const std::string &Str) { auto Loc = StrMap.find(Str); if (Loc != StrMap.end()) return Loc->second; auto S = add(new SPIRVString(this, getId(), Str)); StrMap[Str] = S; return S; } SPIRVMemberName *SPIRVModuleImpl::addMemberName(SPIRVTypeStruct *ST, SPIRVWord MemberNumber, const std::string &Name) { return add(new SPIRVMemberName(ST, MemberNumber, Name)); } void SPIRVModuleImpl::addUnknownStructField(SPIRVTypeStruct *Struct, unsigned I, SPIRVId ID) { UnknownStructFieldMap[Struct].push_back(std::make_pair(I, ID)); } std::istream &operator>>(std::istream &I, SPIRVModule &M) { SPIRVDecoder Decoder(I, M); SPIRVModuleImpl &MI = *static_cast(&M); // Disable automatic capability filling. MI.setAutoAddCapability(false); MI.setAutoAddExtensions(false); SPIRVWord Magic; Decoder >> Magic; if (!M.getErrorLog().checkError(Magic == MagicNumber, SPIRVEC_InvalidModule, "invalid magic number")) { M.setInvalid(); return I; } Decoder >> MI.SPIRVVersion; const bool SPIRVVersionIsKnown = isSPIRVVersionKnown(MI.SPIRVVersion); if (!M.getErrorLog().checkError( SPIRVVersionIsKnown, SPIRVEC_InvalidModule, "unsupported SPIR-V version number '" + to_string(MI.SPIRVVersion) + "'. Range of supported/known SPIR-V " "versions is " + to_string(VersionNumber::MinimumVersion) + " - " + to_string(VersionNumber::MaximumVersion))) { M.setInvalid(); return I; } bool SPIRVVersionIsAllowed = M.isAllowedToUseVersion(MI.SPIRVVersion); if (!M.getErrorLog().checkError( SPIRVVersionIsAllowed, SPIRVEC_InvalidModule, "incorrect SPIR-V version number " + to_string(MI.SPIRVVersion) + " - it conflicts with --spirv-max-version which is set to " + to_string(M.getMaximumAllowedSPIRVVersion()))) { M.setInvalid(); return I; } SPIRVWord Generator = 0; Decoder >> Generator; MI.GeneratorId = Generator >> 16; MI.GeneratorVer = Generator & 0xFFFF; // Bound for Id Decoder >> MI.NextId; Decoder >> MI.InstSchema; if (!M.getErrorLog().checkError(MI.InstSchema == SPIRVISCH_Default, SPIRVEC_InvalidModule, "unsupported instruction schema")) { M.setInvalid(); return I; } while (Decoder.getWordCountAndOpCode() && M.isModuleValid()) { SPIRVEntry *Entry = Decoder.getEntry(); if (Entry != nullptr) M.add(Entry); } MI.resolveUnknownStructFields(); return I; } SPIRVModule *SPIRVModule::createSPIRVModule() { return new SPIRVModuleImpl(); } SPIRVModule *SPIRVModule::createSPIRVModule(const SPIRV::TranslatorOpts &Opts) { return new SPIRVModuleImpl(Opts); } SPIRVValue *SPIRVModuleImpl::getValue(SPIRVId TheId) const { return get(TheId); } SPIRVType *SPIRVModuleImpl::getValueType(SPIRVId TheId) const { return get(TheId)->getType(); } std::vector SPIRVModuleImpl::getValues(const std::vector &IdVec) const { std::vector ValueVec; for (auto I : IdVec) ValueVec.push_back(getValue(I)); return ValueVec; } std::vector SPIRVModuleImpl::getValueTypes(const std::vector &IdVec) const { std::vector TypeVec; for (auto I : IdVec) TypeVec.push_back(getValue(I)->getType()); return TypeVec; } std::vector SPIRVModuleImpl::getIds(const std::vector &ValueVec) const { std::vector IdVec; for (auto I : ValueVec) IdVec.push_back(I->getId()); return IdVec; } std::vector SPIRVModuleImpl::getIds(const std::vector &ValueVec) const { std::vector IdVec; for (auto I : ValueVec) IdVec.push_back(I->getId()); return IdVec; } SPIRVInstTemplateBase * SPIRVModuleImpl::addInstTemplate(Op OC, SPIRVBasicBlock *BB, SPIRVType *Ty) { assert(!Ty || !Ty->isTypeVoid()); SPIRVId Id = Ty ? getId() : SPIRVID_INVALID; auto Ins = SPIRVInstTemplateBase::create(OC, Ty, Id, BB, this); BB->addInstruction(Ins); return Ins; } SPIRVInstTemplateBase * SPIRVModuleImpl::addInstTemplate(Op OC, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) { assert(!Ty || !Ty->isTypeVoid()); SPIRVId Id = Ty ? getId() : SPIRVID_INVALID; auto Ins = SPIRVInstTemplateBase::create(OC, Ty, Id, Ops, BB, this); BB->addInstruction(Ins); return Ins; } void SPIRVModuleImpl::addInstTemplate(SPIRVInstTemplateBase *Ins, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) { assert(!Ty || !Ty->isTypeVoid()); SPIRVId Id = Ty ? getId() : SPIRVID_INVALID; Ins->init(Ty, Id, BB, this); Ins->setOpWordsAndValidate(Ops); BB->addInstruction(Ins); } SPIRVId SPIRVModuleImpl::getExtInstSetId(SPIRVExtInstSetKind Kind) const { assert(Kind < SPIRVEIS_Count && "Unknown extended instruction set!"); auto Res = ExtInstSetIds.find(Kind); assert(Res != ExtInstSetIds.end() && "extended instruction set not found!"); return Res->second; } bool isSpirvBinary(const std::string &Img) { if (Img.size() < sizeof(unsigned)) return false; auto Magic = reinterpret_cast(Img.data()); return *Magic == MagicNumber; } #ifdef _SPIRV_SUPPORT_TEXT_FMT bool convertSpirv(std::istream &IS, std::ostream &OS, std::string &ErrMsg, bool FromText, bool ToText) { auto SaveOpt = SPIRVUseTextFormat; SPIRVUseTextFormat = FromText; // Conversion from/to SPIR-V text representation is a side feature of the // translator which is mostly intended for debug usage. So, this step cannot // be customized to enable/disable particular extensions or restrict/allow // particular SPIR-V versions: all known SPIR-V versions are allowed, all // known SPIR-V extensions are enabled during this conversion SPIRV::TranslatorOpts DefaultOpts; DefaultOpts.enableAllExtensions(); SPIRVModuleImpl M(DefaultOpts); IS >> M; if (M.getError(ErrMsg) != SPIRVEC_Success) { SPIRVUseTextFormat = SaveOpt; return false; } SPIRVUseTextFormat = ToText; OS << M; if (M.getError(ErrMsg) != SPIRVEC_Success) { SPIRVUseTextFormat = SaveOpt; return false; } SPIRVUseTextFormat = SaveOpt; return true; } bool isSpirvText(const std::string &Img) { std::istringstream SS(Img); unsigned Magic = 0; SS >> Magic; if (SS.bad()) return false; return Magic == MagicNumber; } bool convertSpirv(std::string &Input, std::string &Out, std::string &ErrMsg, bool ToText) { auto FromText = isSpirvText(Input); if (ToText == FromText) { Out = Input; return true; } std::istringstream IS(Input); std::ostringstream OS; if (!convertSpirv(IS, OS, ErrMsg, FromText, ToText)) return false; Out = OS.str(); return true; } #endif // _SPIRV_SUPPORT_TEXT_FMT } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVModule.h000066400000000000000000000773701506646577400230160ustar00rootroot00000000000000//===- SPIRVModule.h - Class to represent a SPIR-V module -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Module class for SPIR-V. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVMODULE_H #define SPIRV_LIBSPIRV_SPIRVMODULE_H #include "LLVMSPIRVOpts.h" #include "SPIRVEntry.h" #include "llvm/IR/Metadata.h" #include #include #include #include namespace llvm { class APInt; } // namespace llvm namespace SPIRV { template class SPIRVConstantBase; using SPIRVConstant = SPIRVConstantBase; class SPIRVBasicBlock; class SPIRVEntry; class SPIRVFunction; class SPIRVInstruction; class SPIRVType; class SPIRVTypeArray; class SPIRVTypeBool; class SPIRVTypeFloat; class SPIRVTypeFunction; class SPIRVTypeInt; class SPIRVTypeOpaque; class SPIRVTypePointer; class SPIRVTypeImage; class SPIRVTypeSampler; class SPIRVTypeSampledImage; class SPIRVTypePipeStorage; class SPIRVTypeStruct; class SPIRVTypeVector; class SPIRVTypeVoid; class SPIRVTypeDeviceEvent; class SPIRVTypeQueue; class SPIRVTypePipe; class SPIRVTypeVmeImageINTEL; class SPIRVValue; class SPIRVVariable; class SPIRVDecorateGeneric; class SPIRVDecorationGroup; class SPIRVGroupDecorate; class SPIRVGroupMemberDecorate; class SPIRVGroupDecorateGeneric; class SPIRVInstTemplateBase; class SPIRVAsmTargetINTEL; class SPIRVAsmINTEL; class SPIRVAsmCallINTEL; class SPIRVTypeBufferSurfaceINTEL; class SPIRVTypeTokenINTEL; class SPIRVTypeJointMatrixINTEL; class SPIRVTypeCooperativeMatrixKHR; class SPIRVConditionalCapabilityINTEL; class SPIRVConditionalExtensionINTEL; class SPIRVConditionalEntryPointINTEL; class SPIRVConditionalCopyObjectINTEL; typedef SPIRVBasicBlock SPIRVLabel; struct SPIRVTypeImageDescriptor; class SPIRVModule { public: typedef std::map SPIRVCapMap; typedef std::map, SPIRVConditionalCapabilityINTEL *> SPIRVConditionalCapMap; typedef std::vector SPIRVConditionalEntryPointVec; typedef std::set> SPIRVConditionalExtensionSet; static SPIRVModule *createSPIRVModule(); static SPIRVModule *createSPIRVModule(const SPIRV::TranslatorOpts &); SPIRVModule(); virtual ~SPIRVModule(); // Object query functions virtual bool exist(SPIRVId) const = 0; virtual bool exist(SPIRVId, SPIRVEntry **) const = 0; template T *get(SPIRVId Id) const { return static_cast(getEntry(Id)); } virtual SPIRVEntry *getEntry(SPIRVId) const = 0; virtual bool hasDebugInfo() const = 0; // Error handling functions virtual SPIRVErrorLog &getErrorLog() = 0; virtual SPIRVErrorCode getError(std::string &) = 0; // Check if extension is allowed, and set ErrCode and DetailedMsg if not. // Returns true if no error. virtual bool checkExtension(ExtensionID, SPIRVErrorCode, const std::string &) = 0; void setInvalid() { IsValid = false; } bool isModuleValid() { return IsValid; } // Module query functions virtual SPIRVAddressingModelKind getAddressingModel() = 0; virtual const SPIRVCapMap &getCapability() const = 0; virtual const SPIRVConditionalCapMap &getConditionalCapabilities() const = 0; virtual const SPIRVConditionalEntryPointVec & getConditionalEntryPoints() const = 0; virtual bool hasCapability(SPIRVCapabilityKind) const = 0; virtual SPIRVExtInstSetKind getBuiltinSet(SPIRVId) const = 0; virtual std::set &getExtension() = 0; virtual SPIRVConditionalExtensionSet &getConditionalExtensions() = 0; virtual SPIRVFunction *getFunction(unsigned) const = 0; virtual SPIRVVariable *getVariable(unsigned) const = 0; virtual SPIRVValue *getConst(unsigned) const = 0; virtual std::vector *getDecorateVec() = 0; virtual std::vector *getFuncVec() = 0; virtual SPIRVMemoryModelKind getMemoryModel() const = 0; virtual unsigned getNumFunctions() const = 0; virtual unsigned getNumVariables() const = 0; virtual unsigned getNumConsts() const = 0; virtual std::vector getFunctionPointers() const = 0; virtual SourceLanguage getSourceLanguage(SPIRVWord *) const = 0; virtual std::set &getSourceExtension() = 0; virtual SPIRVValue *getValue(SPIRVId TheId) const = 0; virtual std::vector getValues(const std::vector &) const = 0; virtual std::vector getIds(const std::vector &) const = 0; virtual std::vector getIds(const std::vector &) const = 0; virtual SPIRVType *getValueType(SPIRVId TheId) const = 0; virtual std::vector getValueTypes(const std::vector &) const = 0; virtual SPIRVConstant *getLiteralAsConstant(unsigned Literal) = 0; virtual bool isEntryPoint(SPIRVExecutionModelKind, SPIRVId) const = 0; virtual unsigned short getGeneratorId() const = 0; virtual unsigned short getGeneratorVer() const = 0; virtual SPIRVWord getSPIRVVersion() const = 0; virtual const std::vector &getDebugInstVec() const = 0; virtual const std::vector &getAuxDataInstVec() const = 0; virtual const std::vector &getStringVec() const = 0; // Module changing functions virtual bool importBuiltinSet(const std::string &, SPIRVId *) = 0; virtual bool importBuiltinSetWithId(const std::string &, SPIRVId) = 0; virtual void setAddressingModel(SPIRVAddressingModelKind) = 0; virtual void setAlignment(SPIRVValue *, SPIRVWord) = 0; virtual void setMemoryModel(SPIRVMemoryModelKind) = 0; virtual void setName(SPIRVEntry *, const std::string &) = 0; virtual void setSourceLanguage(SourceLanguage, SPIRVWord) = 0; virtual void setAutoAddCapability(bool E) { AutoAddCapability = E; } virtual void setValidateCapability(bool E) { ValidateCapability = E; } virtual void setAutoAddExtensions(bool E) { AutoAddExtensions = E; } virtual void setGeneratorId(unsigned short) = 0; virtual void setGeneratorVer(unsigned short) = 0; virtual void resolveUnknownStructFields() = 0; virtual void setSPIRVVersion(SPIRVWord) = 0; virtual void insertEntryNoId(SPIRVEntry *Entry) = 0; virtual bool eraseReferencesOfInst(SPIRVId Id) = 0; virtual void eraseCapability(SPIRVCapabilityKind CapKind) = 0; void setMinSPIRVVersion(VersionNumber Ver) { setSPIRVVersion(std::max(static_cast(Ver), getSPIRVVersion())); } // Object creation functions template T *add(T *Entry) { addEntry(Entry); return Entry; } virtual SPIRVEntry *addEntry(SPIRVEntry *) = 0; virtual SPIRVBasicBlock *addBasicBlock(SPIRVFunction *, SPIRVId Id = SPIRVID_INVALID) = 0; virtual SPIRVString *getString(const std::string &Str) = 0; virtual SPIRVMemberName *addMemberName(SPIRVTypeStruct *ST, SPIRVWord MemberNumber, const std::string &Name) = 0; virtual void addUnknownStructField(SPIRVTypeStruct *, unsigned Idx, SPIRVId Id) = 0; virtual void addLine(SPIRVEntry *E, SPIRVId FileNameId, SPIRVWord Line, SPIRVWord Column) = 0; virtual const std::shared_ptr &getCurrentLine() const = 0; virtual void setCurrentLine(const std::shared_ptr &) = 0; virtual void addDebugLine(SPIRVEntry *E, SPIRVType *TheType, SPIRVId FileNameId, SPIRVWord LineStart, SPIRVWord LineEnd, SPIRVWord ColumnStart, SPIRVWord ColumnEnd) = 0; virtual const std::shared_ptr & getCurrentDebugLine() const = 0; virtual void setCurrentDebugLine(const std::shared_ptr &) = 0; virtual const SPIRVDecorateGeneric *addDecorate(SPIRVDecorateGeneric *) = 0; virtual SPIRVDecorationGroup *addDecorationGroup() = 0; virtual SPIRVDecorationGroup * addDecorationGroup(SPIRVDecorationGroup *Group) = 0; virtual SPIRVGroupDecorate * addGroupDecorate(SPIRVDecorationGroup *Group, const std::vector &Targets) = 0; virtual SPIRVGroupMemberDecorate * addGroupMemberDecorate(SPIRVDecorationGroup *Group, const std::vector &Targets) = 0; virtual SPIRVGroupDecorateGeneric * addGroupDecorateGeneric(SPIRVGroupDecorateGeneric *GDec) = 0; virtual void addEntryPoint(SPIRVExecutionModelKind, SPIRVId, const std::string &, const std::vector &) = 0; virtual void addConditionalEntryPoint(SPIRVId, SPIRVExecutionModelKind, SPIRVId, const std::string &, const std::vector &) = 0; virtual void specializeConditionalEntryPoints(SPIRVId, bool) = 0; virtual SPIRVForward *addForward(SPIRVType *Ty) = 0; virtual SPIRVForward *addForward(SPIRVId, SPIRVType *Ty) = 0; virtual SPIRVFunction *addFunction(SPIRVFunction *) = 0; virtual SPIRVFunction *addFunction(SPIRVTypeFunction *, SPIRVId Id = SPIRVID_INVALID) = 0; virtual SPIRVEntry *replaceForward(SPIRVForward *, SPIRVEntry *) = 0; virtual void eraseInstruction(SPIRVInstruction *, SPIRVBasicBlock *) = 0; virtual bool eraseValue(SPIRVValue *) = 0; // Type creation functions virtual SPIRVTypeArray *addArrayType(SPIRVType *, SPIRVConstant *) = 0; virtual SPIRVTypeBool *addBoolType() = 0; virtual SPIRVTypeFloat *addFloatType(unsigned, unsigned = FPEncodingMax) = 0; virtual SPIRVTypeFunction * addFunctionType(SPIRVType *, const std::vector &) = 0; virtual SPIRVTypeImage *addImageType(SPIRVType *, const SPIRVTypeImageDescriptor &) = 0; virtual SPIRVTypeImage *addImageType(SPIRVType *, const SPIRVTypeImageDescriptor &, SPIRVAccessQualifierKind) = 0; virtual SPIRVTypeSampler *addSamplerType() = 0; virtual SPIRVTypePipeStorage *addPipeStorageType() = 0; virtual SPIRVTypeSampledImage *addSampledImageType(SPIRVTypeImage *T) = 0; virtual SPIRVTypeInt *addIntegerType(unsigned) = 0; virtual SPIRVTypeOpaque *addOpaqueType(const std::string &) = 0; virtual SPIRVTypePointer *addPointerType(SPIRVStorageClassKind, SPIRVType *) = 0; virtual SPIRVTypeStruct *openStructType(unsigned, const std::string &) = 0; virtual SPIRVEntry *addTypeStructContinuedINTEL(unsigned NumMembers) = 0; virtual void closeStructType(SPIRVTypeStruct *, bool) = 0; virtual SPIRVTypeVector *addVectorType(SPIRVType *, SPIRVWord) = 0; virtual SPIRVTypeJointMatrixINTEL * addJointMatrixINTELType(SPIRVType *, std::vector) = 0; virtual SPIRVTypeCooperativeMatrixKHR * addCooperativeMatrixKHRType(SPIRVType *, std::vector) = 0; virtual SPIRVInstruction * addCooperativeMatrixLengthKHRInst(SPIRVType *, SPIRVType *, SPIRVBasicBlock *) = 0; virtual SPIRVTypeVoid *addVoidType() = 0; virtual SPIRVType *addOpaqueGenericType(Op) = 0; virtual SPIRVTypeDeviceEvent *addDeviceEventType() = 0; virtual SPIRVTypeQueue *addQueueType() = 0; virtual SPIRVTypePipe *addPipeType() = 0; virtual SPIRVType *addSubgroupAvcINTELType(Op) = 0; virtual SPIRVTypeVmeImageINTEL *addVmeImageINTELType(SPIRVTypeImage *) = 0; virtual SPIRVTypeBufferSurfaceINTEL * addBufferSurfaceINTELType(SPIRVAccessQualifierKind Access) = 0; virtual SPIRVTypeTokenINTEL *addTokenTypeINTEL() = 0; // Constants creation functions virtual SPIRVValue * addCompositeConstant(SPIRVType *, const std::vector &) = 0; virtual SPIRVEntry * addCompositeConstantContinuedINTEL(const std::vector &) = 0; virtual SPIRVValue * addSpecConstantComposite(SPIRVType *Ty, const std::vector &Elements) = 0; virtual SPIRVEntry * addSpecConstantCompositeContinuedINTEL(const std::vector &) = 0; virtual SPIRVValue *addConstantFunctionPointerINTEL(SPIRVType *Ty, SPIRVFunction *F) = 0; virtual SPIRVValue *addConstant(SPIRVValue *) = 0; virtual SPIRVValue *addConstant(SPIRVType *, uint64_t) = 0; virtual SPIRVValue *addConstant(SPIRVType *, llvm::APInt) = 0; virtual SPIRVValue *addSpecConstant(SPIRVType *, uint64_t) = 0; virtual SPIRVValue *addDoubleConstant(SPIRVTypeFloat *, double) = 0; virtual SPIRVValue *addFloatConstant(SPIRVTypeFloat *, float) = 0; virtual SPIRVValue *addIntegerConstant(SPIRVTypeInt *, uint64_t) = 0; virtual SPIRVValue *addNullConstant(SPIRVType *) = 0; virtual SPIRVValue *addUndef(SPIRVType *TheType) = 0; virtual SPIRVValue *addSamplerConstant(SPIRVType *TheType, SPIRVWord AddrMode, SPIRVWord ParametricMode, SPIRVWord FilterMode) = 0; virtual SPIRVValue *addPipeStorageConstant(SPIRVType *TheType, SPIRVWord PacketSize, SPIRVWord PacketAlign, SPIRVWord Capacity) = 0; // Instruction creation functions virtual SPIRVInstruction *addPtrAccessChainInst(SPIRVType *, SPIRVValue *, std::vector, SPIRVBasicBlock *, bool) = 0; virtual SPIRVInstruction * addAsyncGroupCopy(SPIRVValue *Scope, SPIRVValue *Dest, SPIRVValue *Src, SPIRVValue *NumElems, SPIRVValue *Stride, SPIRVValue *Event, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addBinaryInst(Op, SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addBranchConditionalInst(SPIRVValue *, SPIRVLabel *, SPIRVLabel *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addBranchInst(SPIRVLabel *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addExtInst(SPIRVType *, SPIRVWord, SPIRVWord, const std::vector &, SPIRVBasicBlock *, SPIRVInstruction * = nullptr) = 0; virtual SPIRVInstruction *addExtInst(SPIRVType *, SPIRVWord, SPIRVWord, const std::vector &, SPIRVBasicBlock *, SPIRVInstruction * = nullptr) = 0; virtual SPIRVEntry *createDebugInfo(SPIRVWord, SPIRVType *, const std::vector &) = 0; virtual SPIRVEntry *addDebugInfo(SPIRVWord, SPIRVType *, const std::vector &) = 0; virtual SPIRVEntry *addAuxData(SPIRVWord, SPIRVType *, const std::vector &) = 0; virtual SPIRVEntry *addModuleProcessed(const std::string &) = 0; virtual void addCapability(SPIRVCapabilityKind) = 0; template void addCapabilities(const T &Caps) { for (auto I : Caps) addCapability(I); } virtual void addConditionalCapability(SPIRVId, SPIRVCapabilityKind) = 0; template void addConditionalCapabilities(SPIRVId Condition, const T &Caps) { for (auto I : Caps) addConditionalCapability(Condition, I); } virtual void eraseConditionalCapability(SPIRVId, SPIRVCapabilityKind) = 0; virtual void addExtension(ExtensionID) = 0; /// Used by SPIRV entries to add required capability internally. /// Should not be used by users directly. virtual void addCapabilityInternal(SPIRVCapabilityKind) = 0; virtual SPIRVInstruction *addCallInst(SPIRVFunction *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addIndirectCallInst(SPIRVValue *, SPIRVType *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVEntry *getOrAddAsmTargetINTEL(const std::string &) = 0; virtual SPIRVValue *addAsmINTEL(SPIRVTypeFunction *, SPIRVAsmTargetINTEL *, const std::string &, const std::string &) = 0; virtual SPIRVInstruction *addAsmCallINTELInst(SPIRVAsmINTEL *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction * addCompositeConstructInst(SPIRVType *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction * addCompositeExtractInst(SPIRVType *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction * addCompositeInsertInst(SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addCopyObjectInst(SPIRVType *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addCopyMemoryInst(SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction * addCopyMemorySizedInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addCmpInst(Op, SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addControlBarrierInst(SPIRVValue *ExecKind, SPIRVValue *MemKind, SPIRVValue *MemSema, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addGroupInst(Op OpCode, SPIRVType *Type, Scope Scope, const std::vector &Ops, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstTemplateBase *addInstTemplate(Op OC, SPIRVBasicBlock *BB, SPIRVType *Ty) = 0; virtual SPIRVInstTemplateBase * addInstTemplate(Op OC, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) = 0; virtual void addInstTemplate(SPIRVInstTemplateBase *Ins, const std::vector &Ops, SPIRVBasicBlock *BB, SPIRVType *Ty) = 0; virtual SPIRVInstruction *addLoadInst(SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addLifetimeInst(Op OC, SPIRVValue *Object, SPIRVWord Size, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addMemoryBarrierInst(Scope ScopeKind, SPIRVWord MemFlag, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addPhiInst(SPIRVType *, std::vector, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addUnreachableInst(SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addReturnInst(SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addReturnValueInst(SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addSelectInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addSelectionMergeInst(SPIRVId MergeBlock, SPIRVWord SelectionControl, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addLoopMergeInst( SPIRVId MergeBlock, SPIRVId ContinueTarget, SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction * addLoopControlINTELInst(SPIRVWord LoopControl, std::vector LoopControlParameters, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction * addFixedPointIntelInst(Op OC, SPIRVType *ResTy, SPIRVValue *Input, const std::vector &Ops, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction * addArbFloatPointIntelInst(Op OC, SPIRVType *ResTy, SPIRVValue *InA, SPIRVValue *InB, const std::vector &Ops, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addStoreInst(SPIRVValue *, SPIRVValue *, const std::vector &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addSwitchInst( SPIRVValue *, SPIRVBasicBlock *, const std::vector, SPIRVBasicBlock *>> &, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addVectorTimesScalarInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheScalar, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addVectorTimesMatrixInst(SPIRVType *TheType, SPIRVId TheVector, SPIRVId TheMatrix, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addMatrixTimesScalarInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheScalar, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addMatrixTimesVectorInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVId TheVector, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addMatrixTimesMatrixInst(SPIRVType *TheType, SPIRVId M1, SPIRVId M2, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addTransposeInst(SPIRVType *TheType, SPIRVId TheMatrix, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addUnaryInst(Op, SPIRVType *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addVariable(SPIRVType *, bool, SPIRVLinkageTypeKind, SPIRVValue *, const std::string &, SPIRVStorageClassKind, SPIRVBasicBlock *) = 0; virtual SPIRVValue * addVectorShuffleInst(SPIRVType *Type, SPIRVValue *Vec1, SPIRVValue *Vec2, const std::vector &Components, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addVectorExtractDynamicInst(SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addVectorInsertDynamicInst(SPIRVValue *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addFPGARegINTELInst(SPIRVType *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVInstruction *addSampledImageInst(SPIRVType *, SPIRVValue *, SPIRVValue *, SPIRVBasicBlock *) = 0; virtual SPIRVEntry * getOrAddAliasDomainDeclINTELInst(std::vector Args, llvm::MDNode *MD) = 0; virtual SPIRVEntry *getOrAddAliasScopeDeclINTELInst(std::vector Args, llvm::MDNode *MD) = 0; virtual SPIRVEntry * getOrAddAliasScopeListDeclINTELInst(std::vector Args, llvm::MDNode *MD) = 0; virtual SPIRVInstruction *addAssumeTrueKHRInst(SPIRVValue *Condition, SPIRVBasicBlock *BB) = 0; virtual SPIRVInstruction *addExpectKHRInst(SPIRVType *ResultTy, SPIRVValue *Value, SPIRVValue *ExpectedValue, SPIRVBasicBlock *BB) = 0; virtual SPIRVId getExtInstSetId(SPIRVExtInstSetKind Kind) const = 0; virtual bool isAllowedToUseVersion(SPIRV::VersionNumber RequestedVersion) const final { return TranslationOpts.isAllowedToUseVersion(RequestedVersion); } virtual bool isAllowedToUseVersion(SPIRVWord RequestedVersion) const final { return TranslationOpts.isAllowedToUseVersion( static_cast(RequestedVersion)); } virtual SPIRV::VersionNumber getMaximumAllowedSPIRVVersion() const final { return TranslationOpts.getMaxVersion(); } virtual bool isAllowedToUseExtension(ExtensionID RequestedExtension) const final { return TranslationOpts.isAllowedToUseExtension(RequestedExtension); } virtual bool isGenArgNameMDEnabled() const final { return TranslationOpts.isGenArgNameMDEnabled(); } virtual std::vector getModuleProcessedVec() = 0; bool getSpecializationConstant(SPIRVWord SpecId, uint64_t &ConstValue) { return TranslationOpts.getSpecializationConstant(SpecId, ConstValue); } void setSpecializationConstant(SPIRVWord SpecId, uint64_t ConstValue) { TranslationOpts.setSpecConst(SpecId, ConstValue); } FPContractMode getFPContractMode() const { return TranslationOpts.getFPContractMode(); } bool isUnknownIntrinsicAllowed(llvm::IntrinsicInst *II) const noexcept { return TranslationOpts.isUnknownIntrinsicAllowed(II); } bool isSPIRVAllowUnknownIntrinsicsEnabled() const noexcept { return TranslationOpts.isSPIRVAllowUnknownIntrinsicsEnabled(); } bool allowExtraDIExpressions() const noexcept { return TranslationOpts.allowExtraDIExpressions(); } bool shouldReplaceLLVMFmulAddWithOpenCLMad() const noexcept { return TranslationOpts.shouldReplaceLLVMFmulAddWithOpenCLMad(); } bool shouldPreserveOCLKernelArgTypeMetadataThroughString() const noexcept { return TranslationOpts .shouldPreserveOCLKernelArgTypeMetadataThroughString(); } bool shouldEmitFunctionPtrAddrSpace() const noexcept { return TranslationOpts.shouldEmitFunctionPtrAddrSpace(); } bool preserveAuxData() const noexcept { return TranslationOpts.preserveAuxData(); } BuiltinFormat getBuiltinFormat() const noexcept { return TranslationOpts.getBuiltinFormat(); } SPIRVExtInstSetKind getDebugInfoEIS() const { switch (TranslationOpts.getDebugInfoEIS()) { case DebugInfoEIS::SPIRV_Debug: return SPIRVEIS_Debug; case DebugInfoEIS::OpenCL_DebugInfo_100: return SPIRVEIS_OpenCL_DebugInfo_100; case DebugInfoEIS::NonSemantic_Shader_DebugInfo_100: return SPIRVEIS_NonSemantic_Shader_DebugInfo_100; case DebugInfoEIS::NonSemantic_Shader_DebugInfo_200: return SPIRVEIS_NonSemantic_Shader_DebugInfo_200; } assert(false && "Unexpected debug info EIS!"); return SPIRVEIS_Debug; } BIsRepresentation getDesiredBIsRepresentation() const { return TranslationOpts.getDesiredBIsRepresentation(); } llvm::Optional getFnVarCategory() const { return TranslationOpts.getFnVarCategory(); } llvm::Optional getFnVarFamily() const { return TranslationOpts.getFnVarFamily(); } llvm::Optional getFnVarArch() const { return TranslationOpts.getFnVarArch(); } llvm::Optional getFnVarTarget() const { return TranslationOpts.getFnVarTarget(); } std::vector getFnVarFeatures() const { return TranslationOpts.getFnVarFeatures(); } std::vector getFnVarCapabilities() const { return TranslationOpts.getFnVarCapabilities(); } std::string getFnVarSpvOut() const { return TranslationOpts.getFnVarSpvOut(); } // I/O functions friend spv_ostream &operator<<(spv_ostream &O, SPIRVModule &M); friend std::istream &operator>>(std::istream &I, SPIRVModule &M); protected: bool AutoAddCapability; bool ValidateCapability; bool AutoAddExtensions = true; SPIRV::TranslatorOpts TranslationOpts; private: bool IsValid; }; #ifdef _SPIRV_SUPPORT_TEXT_FMT /// Convert SPIR-V between binary and internel text formats. /// This function is not thread safe and should not be used in multi-thread /// applications unless guarded by a critical section. bool ConvertSPIRV(std::istream &IS, spv_ostream &OS, std::string &ErrMsg, bool FromText, bool ToText); /// Convert SPIR-V between binary and internel text formats. /// This function is not thread safe and should not be used in multi-thread /// applications unless guarded by a critical section. bool ConvertSPIRV(std::string &Input, std::string &Out, std::string &ErrMsg, bool ToText); #endif } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVMODULE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h000066400000000000000000001066541506646577400237320ustar00rootroot00000000000000//===- SPIRVNameMapEnum.h - SPIR-V NameMap enums ----------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V NameMap enums. /// //===----------------------------------------------------------------------===// // WARNING: // // This file has been generated using `tools/spirv-tool/gen_spirv.bash` and // should not be modified manually. If the file needs to be updated, edit the // script and any other source file instead, before re-generating this file. //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVNAMEMAPENUM_H #define SPIRV_LIBSPIRV_SPIRVNAMEMAPENUM_H #include "SPIRVEnum.h" #include "spirv/unified1/spirv.hpp" #include "spirv_internal.hpp" using namespace spv; namespace SPIRV { template <> inline void SPIRVMap::init() { add(LinkageTypeExport, "Export"); add(LinkageTypeImport, "Import"); add(LinkageTypeLinkOnceODR, "LinkOnceODR"); add(internal::LinkageTypeInternal, "Internal"); add(LinkageTypeMax, "Max"); } SPIRV_DEF_NAMEMAP(LinkageType, SPIRVLinkageTypeNameMap) template <> inline void SPIRVMap::init() { add(DecorationRelaxedPrecision, "RelaxedPrecision"); add(DecorationSpecId, "SpecId"); add(DecorationBlock, "Block"); add(DecorationBufferBlock, "BufferBlock"); add(DecorationRowMajor, "RowMajor"); add(DecorationColMajor, "ColMajor"); add(DecorationArrayStride, "ArrayStride"); add(DecorationMatrixStride, "MatrixStride"); add(DecorationGLSLShared, "GLSLShared"); add(DecorationGLSLPacked, "GLSLPacked"); add(DecorationCPacked, "CPacked"); add(DecorationBuiltIn, "BuiltIn"); add(DecorationNoPerspective, "NoPerspective"); add(DecorationFlat, "Flat"); add(DecorationPatch, "Patch"); add(DecorationCentroid, "Centroid"); add(DecorationSample, "Sample"); add(DecorationInvariant, "Invariant"); add(DecorationRestrict, "Restrict"); add(DecorationAliased, "Aliased"); add(DecorationVolatile, "Volatile"); add(DecorationConstant, "Constant"); add(DecorationCoherent, "Coherent"); add(DecorationNonWritable, "NonWritable"); add(DecorationNonReadable, "NonReadable"); add(DecorationUniform, "Uniform"); add(DecorationUniformId, "UniformId"); add(DecorationSaturatedConversion, "SaturatedConversion"); add(DecorationStream, "Stream"); add(DecorationLocation, "Location"); add(DecorationComponent, "Component"); add(DecorationIndex, "Index"); add(DecorationBinding, "Binding"); add(DecorationDescriptorSet, "DescriptorSet"); add(DecorationOffset, "Offset"); add(DecorationXfbBuffer, "XfbBuffer"); add(DecorationXfbStride, "XfbStride"); add(DecorationFuncParamAttr, "FuncParamAttr"); add(DecorationFPRoundingMode, "FPRoundingMode"); add(DecorationFPFastMathMode, "FPFastMathMode"); add(DecorationLinkageAttributes, "LinkageAttributes"); add(DecorationNoContraction, "NoContraction"); add(DecorationInputAttachmentIndex, "InputAttachmentIndex"); add(DecorationAlignment, "Alignment"); add(DecorationMaxByteOffset, "MaxByteOffset"); add(DecorationAlignmentId, "AlignmentId"); add(DecorationMaxByteOffsetId, "MaxByteOffsetId"); add(DecorationNoSignedWrap, "NoSignedWrap"); add(DecorationNoUnsignedWrap, "NoUnsignedWrap"); add(DecorationExplicitInterpAMD, "ExplicitInterpAMD"); add(DecorationOverrideCoverageNV, "OverrideCoverageNV"); add(DecorationPassthroughNV, "PassthroughNV"); add(DecorationViewportRelativeNV, "ViewportRelativeNV"); add(DecorationSecondaryViewportRelativeNV, "SecondaryViewportRelativeNV"); add(DecorationPerPrimitiveNV, "PerPrimitiveNV"); add(DecorationPerViewNV, "PerViewNV"); add(DecorationPerTaskNV, "PerTaskNV"); add(DecorationPerVertexKHR, "PerVertexKHR"); add(DecorationPerVertexNV, "PerVertexNV"); add(DecorationNonUniform, "NonUniform"); add(DecorationNonUniformEXT, "NonUniformEXT"); add(DecorationRestrictPointer, "RestrictPointer"); add(DecorationRestrictPointerEXT, "RestrictPointerEXT"); add(DecorationAliasedPointer, "AliasedPointer"); add(DecorationAliasedPointerEXT, "AliasedPointerEXT"); add(DecorationBindlessSamplerNV, "BindlessSamplerNV"); add(DecorationBindlessImageNV, "BindlessImageNV"); add(DecorationBoundSamplerNV, "BoundSamplerNV"); add(DecorationBoundImageNV, "BoundImageNV"); add(DecorationSIMTCallINTEL, "SIMTCallINTEL"); add(DecorationReferencedIndirectlyINTEL, "ReferencedIndirectlyINTEL"); add(DecorationClobberINTEL, "ClobberINTEL"); add(DecorationSideEffectsINTEL, "SideEffectsINTEL"); add(DecorationVectorComputeVariableINTEL, "VectorComputeVariableINTEL"); add(DecorationFuncParamIOKindINTEL, "FuncParamIOKind"); add(DecorationVectorComputeFunctionINTEL, "VectorComputeFunctionINTEL"); add(DecorationStackCallINTEL, "StackCallINTEL"); add(DecorationGlobalVariableOffsetINTEL, "GlobalVariableOffsetINTEL"); add(DecorationCounterBuffer, "CounterBuffer"); add(DecorationHlslCounterBufferGOOGLE, "HlslCounterBufferGOOGLE"); add(DecorationHlslSemanticGOOGLE, "HlslSemanticGOOGLE"); add(DecorationUserSemantic, "UserSemantic"); add(DecorationUserTypeGOOGLE, "UserTypeGOOGLE"); add(DecorationFunctionRoundingModeINTEL, "FunctionRoundingModeINTEL"); add(DecorationFunctionDenormModeINTEL, "FunctionDenormModeINTEL"); add(DecorationRegisterINTEL, "RegisterINTEL"); add(DecorationMemoryINTEL, "MemoryINTEL"); add(DecorationNumbanksINTEL, "NumbanksINTEL"); add(DecorationBankwidthINTEL, "BankwidthINTEL"); add(DecorationMaxPrivateCopiesINTEL, "MaxPrivateCopiesINTEL"); add(DecorationSinglepumpINTEL, "SinglepumpINTEL"); add(DecorationDoublepumpINTEL, "DoublepumpINTEL"); add(DecorationMaxReplicatesINTEL, "MaxReplicatesINTEL"); add(DecorationSimpleDualPortINTEL, "SimpleDualPortINTEL"); add(DecorationMergeINTEL, "MergeINTEL"); add(DecorationBankBitsINTEL, "BankBitsINTEL"); add(DecorationForcePow2DepthINTEL, "ForcePow2DepthINTEL"); add(DecorationBurstCoalesceINTEL, "BurstCoalesceINTEL"); add(DecorationCacheSizeINTEL, "CacheSizeINTEL"); add(DecorationDontStaticallyCoalesceINTEL, "DontStaticallyCoalesceINTEL"); add(DecorationPrefetchINTEL, "PrefetchINTEL"); add(DecorationStallEnableINTEL, "StallEnableINTEL"); add(DecorationFuseLoopsInFunctionINTEL, "FuseLoopsInFunctionINTEL"); add(DecorationAliasScopeINTEL, "AliasScopeINTEL"); add(DecorationNoAliasINTEL, "NoAliasINTEL"); add(DecorationBufferLocationINTEL, "BufferLocationINTEL"); add(DecorationIOPipeStorageINTEL, "IOPipeStorageINTEL"); add(DecorationFunctionFloatingPointModeINTEL, "FunctionFloatingPointModeINTEL"); add(DecorationSingleElementVectorINTEL, "SingleElementVectorINTEL"); add(DecorationVectorComputeCallableFunctionINTEL, "VectorComputeCallableFunctionINTEL"); add(DecorationMediaBlockIOINTEL, "MediaBlockIOINTEL"); add(DecorationAliasScopeINTEL, "AliasScopeINTEL"); add(DecorationNoAliasINTEL, "NoAliasINTEL"); add(DecorationFPMaxErrorDecorationINTEL, "FPMaxErrorDecorationINTEL"); add(DecorationHostAccessINTEL, "HostAccessINTEL"); add(DecorationInitModeINTEL, "InitModeINTEL"); add(DecorationImplementInRegisterMapINTEL, "ImplementInRegisterMapINTEL"); add(DecorationCacheControlLoadINTEL, "CacheControlLoadINTEL"); add(DecorationCacheControlStoreINTEL, "CacheControlStoreINTEL"); add(DecorationConditionalINTEL, "DecorationConditionalINTEL"); // From spirv_internal.hpp add(internal::DecorationMathOpDSPModeINTEL, "MathOpDSPModeINTEL"); add(internal::DecorationInitiationIntervalINTEL, "InitiationIntervalINTEL"); add(internal::DecorationMaxConcurrencyINTEL, "MaxConcurrencyINTEL"); add(internal::DecorationPipelineEnableINTEL, "PipelineEnableINTEL"); add(internal::DecorationRuntimeAlignedINTEL, "RuntimeAlignedINTEL"); add(internal::DecorationHostAccessINTEL, "HostAccessINTEL"); add(internal::DecorationInitModeINTEL, "InitModeINTEL"); add(internal::DecorationImplementInCSRINTEL, "ImplementInCSRINTEL"); add(internal::DecorationArgumentAttributeINTEL, "ArgumentAttributeINTEL"); add(DecorationMax, "Max"); } SPIRV_DEF_NAMEMAP(Decoration, SPIRVDecorationNameMap) template <> inline void SPIRVMap::init() { add(BuiltInPosition, "BuiltInPosition"); add(BuiltInPointSize, "BuiltInPointSize"); add(BuiltInClipDistance, "BuiltInClipDistance"); add(BuiltInCullDistance, "BuiltInCullDistance"); add(BuiltInVertexId, "BuiltInVertexId"); add(BuiltInInstanceId, "BuiltInInstanceId"); add(BuiltInPrimitiveId, "BuiltInPrimitiveId"); add(BuiltInInvocationId, "BuiltInInvocationId"); add(BuiltInLayer, "BuiltInLayer"); add(BuiltInViewportIndex, "BuiltInViewportIndex"); add(BuiltInTessLevelOuter, "BuiltInTessLevelOuter"); add(BuiltInTessLevelInner, "BuiltInTessLevelInner"); add(BuiltInTessCoord, "BuiltInTessCoord"); add(BuiltInPatchVertices, "BuiltInPatchVertices"); add(BuiltInFragCoord, "BuiltInFragCoord"); add(BuiltInPointCoord, "BuiltInPointCoord"); add(BuiltInFrontFacing, "BuiltInFrontFacing"); add(BuiltInSampleId, "BuiltInSampleId"); add(BuiltInSamplePosition, "BuiltInSamplePosition"); add(BuiltInSampleMask, "BuiltInSampleMask"); add(BuiltInFragDepth, "BuiltInFragDepth"); add(BuiltInHelperInvocation, "BuiltInHelperInvocation"); add(BuiltInNumWorkgroups, "BuiltInNumWorkgroups"); add(BuiltInWorkgroupSize, "BuiltInWorkgroupSize"); add(BuiltInWorkgroupId, "BuiltInWorkgroupId"); add(BuiltInLocalInvocationId, "BuiltInLocalInvocationId"); add(BuiltInGlobalInvocationId, "BuiltInGlobalInvocationId"); add(BuiltInLocalInvocationIndex, "BuiltInLocalInvocationIndex"); add(BuiltInWorkDim, "BuiltInWorkDim"); add(BuiltInGlobalSize, "BuiltInGlobalSize"); add(BuiltInEnqueuedWorkgroupSize, "BuiltInEnqueuedWorkgroupSize"); add(BuiltInGlobalOffset, "BuiltInGlobalOffset"); add(BuiltInGlobalLinearId, "BuiltInGlobalLinearId"); add(BuiltInSubgroupSize, "BuiltInSubgroupSize"); add(BuiltInSubgroupMaxSize, "BuiltInSubgroupMaxSize"); add(BuiltInNumSubgroups, "BuiltInNumSubgroups"); add(BuiltInNumEnqueuedSubgroups, "BuiltInNumEnqueuedSubgroups"); add(BuiltInSubgroupId, "BuiltInSubgroupId"); add(BuiltInSubgroupLocalInvocationId, "BuiltInSubgroupLocalInvocationId"); add(BuiltInVertexIndex, "BuiltInVertexIndex"); add(BuiltInInstanceIndex, "BuiltInInstanceIndex"); add(BuiltInSubgroupEqMask, "BuiltInSubgroupEqMask"); add(BuiltInSubgroupEqMaskKHR, "BuiltInSubgroupEqMaskKHR"); add(BuiltInSubgroupGeMask, "BuiltInSubgroupGeMask"); add(BuiltInSubgroupGeMaskKHR, "BuiltInSubgroupGeMaskKHR"); add(BuiltInSubgroupGtMask, "BuiltInSubgroupGtMask"); add(BuiltInSubgroupGtMaskKHR, "BuiltInSubgroupGtMaskKHR"); add(BuiltInSubgroupLeMask, "BuiltInSubgroupLeMask"); add(BuiltInSubgroupLeMaskKHR, "BuiltInSubgroupLeMaskKHR"); add(BuiltInSubgroupLtMask, "BuiltInSubgroupLtMask"); add(BuiltInSubgroupLtMaskKHR, "BuiltInSubgroupLtMaskKHR"); add(BuiltInBaseVertex, "BuiltInBaseVertex"); add(BuiltInBaseInstance, "BuiltInBaseInstance"); add(BuiltInDrawIndex, "BuiltInDrawIndex"); add(BuiltInPrimitiveShadingRateKHR, "BuiltInPrimitiveShadingRateKHR"); add(BuiltInDeviceIndex, "BuiltInDeviceIndex"); add(BuiltInViewIndex, "BuiltInViewIndex"); add(BuiltInShadingRateKHR, "BuiltInShadingRateKHR"); add(BuiltInBaryCoordNoPerspAMD, "BuiltInBaryCoordNoPerspAMD"); add(BuiltInBaryCoordNoPerspCentroidAMD, "BuiltInBaryCoordNoPerspCentroidAMD"); add(BuiltInBaryCoordNoPerspSampleAMD, "BuiltInBaryCoordNoPerspSampleAMD"); add(BuiltInBaryCoordSmoothAMD, "BuiltInBaryCoordSmoothAMD"); add(BuiltInBaryCoordSmoothCentroidAMD, "BuiltInBaryCoordSmoothCentroidAMD"); add(BuiltInBaryCoordSmoothSampleAMD, "BuiltInBaryCoordSmoothSampleAMD"); add(BuiltInBaryCoordPullModelAMD, "BuiltInBaryCoordPullModelAMD"); add(BuiltInFragStencilRefEXT, "BuiltInFragStencilRefEXT"); add(BuiltInViewportMaskNV, "BuiltInViewportMaskNV"); add(BuiltInSecondaryPositionNV, "BuiltInSecondaryPositionNV"); add(BuiltInSecondaryViewportMaskNV, "BuiltInSecondaryViewportMaskNV"); add(BuiltInPositionPerViewNV, "BuiltInPositionPerViewNV"); add(BuiltInViewportMaskPerViewNV, "BuiltInViewportMaskPerViewNV"); add(BuiltInFullyCoveredEXT, "BuiltInFullyCoveredEXT"); add(BuiltInTaskCountNV, "BuiltInTaskCountNV"); add(BuiltInPrimitiveCountNV, "BuiltInPrimitiveCountNV"); add(BuiltInPrimitiveIndicesNV, "BuiltInPrimitiveIndicesNV"); add(BuiltInClipDistancePerViewNV, "BuiltInClipDistancePerViewNV"); add(BuiltInCullDistancePerViewNV, "BuiltInCullDistancePerViewNV"); add(BuiltInLayerPerViewNV, "BuiltInLayerPerViewNV"); add(BuiltInMeshViewCountNV, "BuiltInMeshViewCountNV"); add(BuiltInMeshViewIndicesNV, "BuiltInMeshViewIndicesNV"); add(BuiltInBaryCoordKHR, "BuiltInBaryCoordKHR"); add(BuiltInBaryCoordNV, "BuiltInBaryCoordNV"); add(BuiltInBaryCoordNoPerspKHR, "BuiltInBaryCoordNoPerspKHR"); add(BuiltInBaryCoordNoPerspNV, "BuiltInBaryCoordNoPerspNV"); add(BuiltInFragSizeEXT, "BuiltInFragSizeEXT"); add(BuiltInFragmentSizeNV, "BuiltInFragmentSizeNV"); add(BuiltInFragInvocationCountEXT, "BuiltInFragInvocationCountEXT"); add(BuiltInInvocationsPerPixelNV, "BuiltInInvocationsPerPixelNV"); add(BuiltInLaunchIdKHR, "BuiltInLaunchIdKHR"); add(BuiltInLaunchIdNV, "BuiltInLaunchIdNV"); add(BuiltInLaunchSizeKHR, "BuiltInLaunchSizeKHR"); add(BuiltInLaunchSizeNV, "BuiltInLaunchSizeNV"); add(BuiltInWorldRayOriginKHR, "BuiltInWorldRayOriginKHR"); add(BuiltInWorldRayOriginNV, "BuiltInWorldRayOriginNV"); add(BuiltInWorldRayDirectionKHR, "BuiltInWorldRayDirectionKHR"); add(BuiltInWorldRayDirectionNV, "BuiltInWorldRayDirectionNV"); add(BuiltInObjectRayOriginKHR, "BuiltInObjectRayOriginKHR"); add(BuiltInObjectRayOriginNV, "BuiltInObjectRayOriginNV"); add(BuiltInObjectRayDirectionKHR, "BuiltInObjectRayDirectionKHR"); add(BuiltInObjectRayDirectionNV, "BuiltInObjectRayDirectionNV"); add(BuiltInRayTminKHR, "BuiltInRayTminKHR"); add(BuiltInRayTminNV, "BuiltInRayTminNV"); add(BuiltInRayTmaxKHR, "BuiltInRayTmaxKHR"); add(BuiltInRayTmaxNV, "BuiltInRayTmaxNV"); add(BuiltInInstanceCustomIndexKHR, "BuiltInInstanceCustomIndexKHR"); add(BuiltInInstanceCustomIndexNV, "BuiltInInstanceCustomIndexNV"); add(BuiltInObjectToWorldKHR, "BuiltInObjectToWorldKHR"); add(BuiltInObjectToWorldNV, "BuiltInObjectToWorldNV"); add(BuiltInWorldToObjectKHR, "BuiltInWorldToObjectKHR"); add(BuiltInWorldToObjectNV, "BuiltInWorldToObjectNV"); add(BuiltInHitTNV, "BuiltInHitTNV"); add(BuiltInHitKindKHR, "BuiltInHitKindKHR"); add(BuiltInHitKindNV, "BuiltInHitKindNV"); add(BuiltInCurrentRayTimeNV, "BuiltInCurrentRayTimeNV"); add(BuiltInIncomingRayFlagsKHR, "BuiltInIncomingRayFlagsKHR"); add(BuiltInIncomingRayFlagsNV, "BuiltInIncomingRayFlagsNV"); add(BuiltInRayGeometryIndexKHR, "BuiltInRayGeometryIndexKHR"); add(BuiltInWarpsPerSMNV, "BuiltInWarpsPerSMNV"); add(BuiltInSMCountNV, "BuiltInSMCountNV"); add(BuiltInWarpIDNV, "BuiltInWarpIDNV"); add(BuiltInSMIDNV, "BuiltInSMIDNV"); add(BuiltInCullMaskKHR, "BuiltInCullMaskKHR"); add(BuiltInMax, "BuiltInMax"); add(internal::BuiltInSubDeviceIDINTEL, "BuiltInSubDeviceIDINTEL"); add(internal::BuiltInGlobalHWThreadIDINTEL, "BuiltInGlobalHWThreadIDINTEL"); } SPIRV_DEF_NAMEMAP(BuiltIn, SPIRVBuiltInNameMap) template <> inline void SPIRVMap::init() { add(CapabilityMatrix, "Matrix"); add(CapabilityShader, "Shader"); add(CapabilityGeometry, "Geometry"); add(CapabilityTessellation, "Tessellation"); add(CapabilityAddresses, "Addresses"); add(CapabilityLinkage, "Linkage"); add(CapabilityKernel, "Kernel"); add(CapabilityVector16, "Vector16"); add(CapabilityFloat16Buffer, "Float16Buffer"); add(CapabilityFloat16, "Float16"); add(CapabilityFloat64, "Float64"); add(CapabilityInt64, "Int64"); add(CapabilityInt64Atomics, "Int64Atomics"); add(CapabilityImageBasic, "ImageBasic"); add(CapabilityImageReadWrite, "ImageReadWrite"); add(CapabilityImageMipmap, "ImageMipmap"); add(CapabilityPipes, "Pipes"); add(CapabilityGroups, "Groups"); add(CapabilityDeviceEnqueue, "DeviceEnqueue"); add(CapabilityLiteralSampler, "LiteralSampler"); add(CapabilityAtomicStorage, "AtomicStorage"); add(CapabilityInt16, "Int16"); add(CapabilityTessellationPointSize, "TessellationPointSize"); add(CapabilityGeometryPointSize, "GeometryPointSize"); add(CapabilityImageGatherExtended, "ImageGatherExtended"); add(CapabilityStorageImageMultisample, "StorageImageMultisample"); add(CapabilityUniformBufferArrayDynamicIndexing, "UniformBufferArrayDynamicIndexing"); add(CapabilitySampledImageArrayDynamicIndexing, "SampledImageArrayDynamicIndexing"); add(CapabilityStorageBufferArrayDynamicIndexing, "StorageBufferArrayDynamicIndexing"); add(CapabilityStorageImageArrayDynamicIndexing, "StorageImageArrayDynamicIndexing"); add(CapabilityClipDistance, "ClipDistance"); add(CapabilityCullDistance, "CullDistance"); add(CapabilityImageCubeArray, "ImageCubeArray"); add(CapabilitySampleRateShading, "SampleRateShading"); add(CapabilityImageRect, "ImageRect"); add(CapabilitySampledRect, "SampledRect"); add(CapabilityGenericPointer, "GenericPointer"); add(CapabilityInt8, "Int8"); add(CapabilityInputAttachment, "InputAttachment"); add(CapabilitySparseResidency, "SparseResidency"); add(CapabilityMinLod, "MinLod"); add(CapabilitySampled1D, "Sampled1D"); add(CapabilityImage1D, "Image1D"); add(CapabilitySampledCubeArray, "SampledCubeArray"); add(CapabilitySampledBuffer, "SampledBuffer"); add(CapabilityImageBuffer, "ImageBuffer"); add(CapabilityImageMSArray, "ImageMSArray"); add(CapabilityStorageImageExtendedFormats, "StorageImageExtendedFormats"); add(CapabilityImageQuery, "ImageQuery"); add(CapabilityDerivativeControl, "DerivativeControl"); add(CapabilityInterpolationFunction, "InterpolationFunction"); add(CapabilityTransformFeedback, "TransformFeedback"); add(CapabilityGeometryStreams, "GeometryStreams"); add(CapabilityStorageImageReadWithoutFormat, "StorageImageReadWithoutFormat"); add(CapabilityStorageImageWriteWithoutFormat, "StorageImageWriteWithoutFormat"); add(CapabilityMultiViewport, "MultiViewport"); add(CapabilitySubgroupDispatch, "SubgroupDispatch"); add(CapabilityNamedBarrier, "NamedBarrier"); add(CapabilityPipeStorage, "PipeStorage"); add(CapabilityGroupNonUniform, "GroupNonUniform"); add(CapabilityGroupNonUniformVote, "GroupNonUniformVote"); add(CapabilityGroupNonUniformArithmetic, "GroupNonUniformArithmetic"); add(CapabilityGroupNonUniformBallot, "GroupNonUniformBallot"); add(CapabilityGroupNonUniformShuffle, "GroupNonUniformShuffle"); add(CapabilityGroupNonUniformShuffleRelative, "GroupNonUniformShuffleRelative"); add(CapabilityGroupNonUniformClustered, "GroupNonUniformClustered"); add(CapabilityGroupNonUniformQuad, "GroupNonUniformQuad"); add(CapabilityShaderLayer, "ShaderLayer"); add(CapabilityShaderViewportIndex, "ShaderViewportIndex"); add(CapabilityUniformDecoration, "UniformDecoration"); add(CapabilityFragmentShadingRateKHR, "FragmentShadingRateKHR"); add(CapabilitySubgroupBallotKHR, "SubgroupBallotKHR"); add(CapabilityDrawParameters, "DrawParameters"); add(CapabilityWorkgroupMemoryExplicitLayoutKHR, "WorkgroupMemoryExplicitLayoutKHR"); add(CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR, "WorkgroupMemoryExplicitLayout8BitAccessKHR"); add(CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR, "WorkgroupMemoryExplicitLayout16BitAccessKHR"); add(CapabilitySubgroupVoteKHR, "SubgroupVoteKHR"); add(CapabilityStorageBuffer16BitAccess, "StorageBuffer16BitAccess"); add(CapabilityStorageUniformBufferBlock16, "StorageUniformBufferBlock16"); add(CapabilityStorageUniform16, "StorageUniform16"); add(CapabilityUniformAndStorageBuffer16BitAccess, "UniformAndStorageBuffer16BitAccess"); add(CapabilityStoragePushConstant16, "StoragePushConstant16"); add(CapabilityStorageInputOutput16, "StorageInputOutput16"); add(CapabilityDeviceGroup, "DeviceGroup"); add(CapabilityMultiView, "MultiView"); add(CapabilityVariablePointersStorageBuffer, "VariablePointersStorageBuffer"); add(CapabilityVariablePointers, "VariablePointers"); add(CapabilityAtomicStorageOps, "AtomicStorageOps"); add(CapabilitySampleMaskPostDepthCoverage, "SampleMaskPostDepthCoverage"); add(CapabilityStorageBuffer8BitAccess, "StorageBuffer8BitAccess"); add(CapabilityUniformAndStorageBuffer8BitAccess, "UniformAndStorageBuffer8BitAccess"); add(CapabilityStoragePushConstant8, "StoragePushConstant8"); add(CapabilityDenormPreserve, "DenormPreserve"); add(CapabilityDenormFlushToZero, "DenormFlushToZero"); add(CapabilitySignedZeroInfNanPreserve, "SignedZeroInfNanPreserve"); add(CapabilityRoundingModeRTE, "RoundingModeRTE"); add(CapabilityRoundingModeRTZ, "RoundingModeRTZ"); add(CapabilityRayQueryProvisionalKHR, "RayQueryProvisionalKHR"); add(CapabilityRayQueryKHR, "RayQueryKHR"); add(CapabilityRayTraversalPrimitiveCullingKHR, "RayTraversalPrimitiveCullingKHR"); add(CapabilityRayTracingKHR, "RayTracingKHR"); add(CapabilityFloat16ImageAMD, "Float16ImageAMD"); add(CapabilityImageGatherBiasLodAMD, "ImageGatherBiasLodAMD"); add(CapabilityFragmentMaskAMD, "FragmentMaskAMD"); add(CapabilityStencilExportEXT, "StencilExportEXT"); add(CapabilityImageReadWriteLodAMD, "ImageReadWriteLodAMD"); add(CapabilityInt64ImageEXT, "Int64ImageEXT"); add(CapabilityShaderClockKHR, "ShaderClockKHR"); add(CapabilitySampleMaskOverrideCoverageNV, "SampleMaskOverrideCoverageNV"); add(CapabilityGeometryShaderPassthroughNV, "GeometryShaderPassthroughNV"); add(CapabilityShaderViewportIndexLayerEXT, "ShaderViewportIndexLayerEXT"); add(CapabilityShaderViewportIndexLayerNV, "ShaderViewportIndexLayerNV"); add(CapabilityShaderViewportMaskNV, "ShaderViewportMaskNV"); add(CapabilityShaderStereoViewNV, "ShaderStereoViewNV"); add(CapabilityPerViewAttributesNV, "PerViewAttributesNV"); add(CapabilityFragmentFullyCoveredEXT, "FragmentFullyCoveredEXT"); add(CapabilityMeshShadingNV, "MeshShadingNV"); add(CapabilityImageFootprintNV, "ImageFootprintNV"); add(CapabilityFragmentBarycentricKHR, "FragmentBarycentricKHR"); add(CapabilityFragmentBarycentricNV, "FragmentBarycentricNV"); add(CapabilityComputeDerivativeGroupQuadsNV, "ComputeDerivativeGroupQuadsNV"); add(CapabilityFragmentDensityEXT, "FragmentDensityEXT"); add(CapabilityShadingRateNV, "ShadingRateNV"); add(CapabilityGroupNonUniformPartitionedNV, "GroupNonUniformPartitionedNV"); add(CapabilityShaderNonUniform, "ShaderNonUniform"); add(CapabilityShaderNonUniformEXT, "ShaderNonUniformEXT"); add(CapabilityRuntimeDescriptorArray, "RuntimeDescriptorArray"); add(CapabilityRuntimeDescriptorArrayEXT, "RuntimeDescriptorArrayEXT"); add(CapabilityInputAttachmentArrayDynamicIndexing, "InputAttachmentArrayDynamicIndexing"); add(CapabilityInputAttachmentArrayDynamicIndexingEXT, "InputAttachmentArrayDynamicIndexingEXT"); add(CapabilityUniformTexelBufferArrayDynamicIndexing, "UniformTexelBufferArrayDynamicIndexing"); add(CapabilityUniformTexelBufferArrayDynamicIndexingEXT, "UniformTexelBufferArrayDynamicIndexingEXT"); add(CapabilityStorageTexelBufferArrayDynamicIndexing, "StorageTexelBufferArrayDynamicIndexing"); add(CapabilityStorageTexelBufferArrayDynamicIndexingEXT, "StorageTexelBufferArrayDynamicIndexingEXT"); add(CapabilityUniformBufferArrayNonUniformIndexing, "UniformBufferArrayNonUniformIndexing"); add(CapabilityUniformBufferArrayNonUniformIndexingEXT, "UniformBufferArrayNonUniformIndexingEXT"); add(CapabilitySampledImageArrayNonUniformIndexing, "SampledImageArrayNonUniformIndexing"); add(CapabilitySampledImageArrayNonUniformIndexingEXT, "SampledImageArrayNonUniformIndexingEXT"); add(CapabilityStorageBufferArrayNonUniformIndexing, "StorageBufferArrayNonUniformIndexing"); add(CapabilityStorageBufferArrayNonUniformIndexingEXT, "StorageBufferArrayNonUniformIndexingEXT"); add(CapabilityStorageImageArrayNonUniformIndexing, "StorageImageArrayNonUniformIndexing"); add(CapabilityStorageImageArrayNonUniformIndexingEXT, "StorageImageArrayNonUniformIndexingEXT"); add(CapabilityInputAttachmentArrayNonUniformIndexing, "InputAttachmentArrayNonUniformIndexing"); add(CapabilityInputAttachmentArrayNonUniformIndexingEXT, "InputAttachmentArrayNonUniformIndexingEXT"); add(CapabilityUniformTexelBufferArrayNonUniformIndexing, "UniformTexelBufferArrayNonUniformIndexing"); add(CapabilityUniformTexelBufferArrayNonUniformIndexingEXT, "UniformTexelBufferArrayNonUniformIndexingEXT"); add(CapabilityStorageTexelBufferArrayNonUniformIndexing, "StorageTexelBufferArrayNonUniformIndexing"); add(CapabilityStorageTexelBufferArrayNonUniformIndexingEXT, "StorageTexelBufferArrayNonUniformIndexingEXT"); add(CapabilityRayTracingNV, "RayTracingNV"); add(CapabilityRayTracingMotionBlurNV, "RayTracingMotionBlurNV"); add(CapabilityVulkanMemoryModel, "VulkanMemoryModel"); add(CapabilityVulkanMemoryModelKHR, "VulkanMemoryModelKHR"); add(CapabilityVulkanMemoryModelDeviceScope, "VulkanMemoryModelDeviceScope"); add(CapabilityVulkanMemoryModelDeviceScopeKHR, "VulkanMemoryModelDeviceScopeKHR"); add(CapabilityPhysicalStorageBufferAddresses, "PhysicalStorageBufferAddresses"); add(CapabilityPhysicalStorageBufferAddressesEXT, "PhysicalStorageBufferAddressesEXT"); add(CapabilityComputeDerivativeGroupLinearNV, "ComputeDerivativeGroupLinearNV"); add(CapabilityRayTracingProvisionalKHR, "RayTracingProvisionalKHR"); add(CapabilityCooperativeMatrixNV, "CooperativeMatrixNV"); add(CapabilityFragmentShaderSampleInterlockEXT, "FragmentShaderSampleInterlockEXT"); add(CapabilityFragmentShaderShadingRateInterlockEXT, "FragmentShaderShadingRateInterlockEXT"); add(CapabilityShaderSMBuiltinsNV, "ShaderSMBuiltinsNV"); add(CapabilityFragmentShaderPixelInterlockEXT, "FragmentShaderPixelInterlockEXT"); add(CapabilityDemoteToHelperInvocation, "DemoteToHelperInvocation"); add(CapabilityDemoteToHelperInvocationEXT, "DemoteToHelperInvocationEXT"); add(CapabilityBindlessTextureNV, "BindlessTextureNV"); add(CapabilitySubgroupShuffleINTEL, "SubgroupShuffleINTEL"); add(CapabilitySubgroupBufferBlockIOINTEL, "SubgroupBufferBlockIOINTEL"); add(CapabilitySubgroupImageBlockIOINTEL, "SubgroupImageBlockIOINTEL"); add(CapabilitySubgroupImageMediaBlockIOINTEL, "SubgroupImageMediaBlockIOINTEL"); add(CapabilityRoundToInfinityINTEL, "RoundToInfinityINTEL"); add(CapabilityFloatingPointModeINTEL, "FloatingPointModeINTEL"); add(CapabilityIntegerFunctions2INTEL, "IntegerFunctions2INTEL"); add(CapabilityFunctionPointersINTEL, "FunctionPointersINTEL"); add(CapabilityIndirectReferencesINTEL, "IndirectReferencesINTEL"); add(CapabilityAsmINTEL, "AsmINTEL"); add(CapabilityAtomicFloat32MinMaxEXT, "AtomicFloat32MinMaxEXT"); add(CapabilityAtomicFloat64MinMaxEXT, "AtomicFloat64MinMaxEXT"); add(CapabilityAtomicFloat16MinMaxEXT, "AtomicFloat16MinMaxEXT"); add(CapabilityVectorComputeINTEL, "VectorComputeINTEL"); add(CapabilityVectorAnyINTEL, "VectorAnyINTEL"); add(CapabilityExpectAssumeKHR, "ExpectAssumeKHR"); add(CapabilitySubgroupAvcMotionEstimationINTEL, "SubgroupAvcMotionEstimationINTEL"); add(CapabilitySubgroupAvcMotionEstimationIntraINTEL, "SubgroupAvcMotionEstimationIntraINTEL"); add(CapabilitySubgroupAvcMotionEstimationChromaINTEL, "SubgroupAvcMotionEstimationChromaINTEL"); add(CapabilityVariableLengthArrayINTEL, "VariableLengthArrayINTEL"); add(CapabilityFunctionFloatControlINTEL, "FunctionFloatControlINTEL"); add(CapabilityFPGAMemoryAttributesINTEL, "FPGAMemoryAttributesINTEL"); add(CapabilityFPFastMathModeINTEL, "FPFastMathModeINTEL"); add(CapabilityArbitraryPrecisionIntegersINTEL, "ArbitraryPrecisionIntegersINTEL"); add(CapabilityArbitraryPrecisionFloatingPointINTEL, "ArbitraryPrecisionFloatingPointINTEL"); add(CapabilityUnstructuredLoopControlsINTEL, "UnstructuredLoopControlsINTEL"); add(CapabilityFPGALoopControlsINTEL, "FPGALoopControlsINTEL"); add(CapabilityKernelAttributesINTEL, "KernelAttributesINTEL"); add(CapabilityFPGAKernelAttributesINTEL, "FPGAKernelAttributesINTEL"); add(CapabilityFPGAMemoryAccessesINTEL, "FPGAMemoryAccessesINTEL"); add(CapabilityFPGAClusterAttributesINTEL, "FPGAClusterAttributesINTEL"); add(CapabilityLoopFuseINTEL, "LoopFuseINTEL"); add(CapabilityMemoryAccessAliasingINTEL, "MemoryAccessAliasingINTEL"); add(CapabilityFPGABufferLocationINTEL, "FPGABufferLocationINTEL"); add(CapabilityArbitraryPrecisionFixedPointINTEL, "ArbitraryPrecisionFixedPointINTEL"); add(CapabilityUSMStorageClassesINTEL, "USMStorageClassesINTEL"); add(CapabilityIOPipesINTEL, "IOPipeINTEL"); add(CapabilityBlockingPipesINTEL, "BlockingPipesINTEL"); add(CapabilityFPGARegINTEL, "FPGARegINTEL"); add(CapabilityDotProductInputAll, "DotProductInputAll"); add(CapabilityDotProductInputAllKHR, "DotProductInputAllKHR"); add(CapabilityDotProductInput4x8Bit, "DotProductInput4x8Bit"); add(CapabilityDotProductInput4x8BitKHR, "DotProductInput4x8BitKHR"); add(CapabilityDotProductInput4x8BitPacked, "DotProductInput4x8BitPacked"); add(CapabilityDotProductInput4x8BitPackedKHR, "DotProductInput4x8BitPackedKHR"); add(CapabilityDotProduct, "DotProduct"); add(CapabilityDotProductKHR, "DotProductKHR"); add(CapabilityRayCullMaskKHR, "RayCullMaskKHR"); add(CapabilityCooperativeMatrixKHR, "CooperativeMatrixKHR"); add(CapabilityBitInstructions, "BitInstructions"); add(CapabilityGroupNonUniformRotateKHR, "GroupNonUniformRotateKHR"); add(CapabilityAtomicFloat32AddEXT, "AtomicFloat32AddEXT"); add(CapabilityAtomicFloat64AddEXT, "AtomicFloat64AddEXT"); add(CapabilityLongCompositesINTEL, "LongCompositesINTEL"); add(CapabilityOptNoneEXT, "OptNoneEXT"); add(CapabilityAtomicFloat16AddEXT, "AtomicFloat16AddEXT"); add(internal::CapabilityAtomicBFloat16AddINTEL, "AtomicBFloat16AddINTEL"); add(internal::CapabilityAtomicBFloat16MinMaxINTEL, "AtomicBFloat16MinMaxINTEL"); add(CapabilityDebugInfoModuleINTEL, "DebugInfoModuleINTEL"); add(CapabilitySplitBarrierINTEL, "SplitBarrierINTEL"); add(CapabilityGlobalVariableFPGADecorationsINTEL, "GlobalVariableFPGADecorationsINTEL"); add(CapabilityGlobalVariableHostAccessINTEL, "GlobalVariableHostAccessINTEL"); add(CapabilityGroupUniformArithmeticKHR, "GroupUniformArithmeticKHR"); add(CapabilityRegisterLimitsINTEL, "RegisterLimitsINTEL"); add(CapabilityFPMaxErrorINTEL, "FPMaxErrorINTEL"); add(CapabilityBFloat16TypeKHR, "BFloat16TypeKHR"); add(CapabilityBFloat16DotProductKHR, "BFloat16DotProductKHR"); add(CapabilityBFloat16CooperativeMatrixKHR, "BFloat16CooperativeMatrixKHR"); add(CapabilitySubgroup2DBlockIOINTEL, "Subgroup2DBlockIOINTEL"); add(CapabilitySubgroup2DBlockTransformINTEL, "Subgroup2DBlockTransformINTEL"); add(CapabilitySubgroup2DBlockTransposeINTEL, "Subgroup2DBlockTransposeINTEL"); add(CapabilitySubgroupMatrixMultiplyAccumulateINTEL, "SubgroupMatrixMultiplyAccumulateINTEL"); add(CapabilityArithmeticFenceEXT, "ArithmeticFenceEXT"); add(CapabilityTernaryBitwiseFunctionINTEL, "TernaryBitwiseFunctionINTEL"); // From spirv_internal.hpp add(internal::CapabilityFPGADSPControlINTEL, "FPGADSPControlINTEL"); add(internal::CapabilityFPGAInvocationPipeliningAttributesINTEL, "FPGAInvocationPipeliningAttributesINTEL"); add(internal::CapabilityTokenTypeINTEL, "TokenTypeINTEL"); add(internal::CapabilityRuntimeAlignedAttributeINTEL, "RuntimeAlignedAttributeINTEL"); add(CapabilityMax, "Max"); add(internal::CapabilityBfloat16ConversionINTEL, "Bfloat16ConversionINTEL"); add(internal::CapabilityJointMatrixINTEL, "JointMatrixINTEL"); add(internal::CapabilityHWThreadQueryINTEL, "HWThreadQueryINTEL"); add(internal::CapabilityGlobalVariableDecorationsINTEL, "GlobalVariableDecorationsINTEL"); add(internal::CapabilityComplexFloatMulDivINTEL, "ComplexFloatMulDivINTEL"); add(internal::CapabilityMaskedGatherScatterINTEL, "MaskedGatherScatterINTEL"); add(internal::CapabilityTensorFloat32RoundingINTEL, "TensorFloat32RoundingINTEL"); add(internal::CapabilityCacheControlsINTEL, "CacheControlsINTEL"); add(internal::CapabilityJointMatrixWIInstructionsINTEL, "JointMatrixWIInstructionsINTEL"); add(internal::CapabilityCooperativeMatrixPrefetchINTEL, "CooperativeMatrixPrefetchINTEL"); add(internal::CapabilityCooperativeMatrixCheckedInstructionsINTEL, "CooperativeMatrixCheckedInstructionsINTEL"); add(internal::CapabilityBindlessImagesINTEL, "BindlessImagesINTEL"); add(CapabilityInt4TypeINTEL, "Int4TypeINTEL"); add(CapabilityInt4CooperativeMatrixINTEL, "Int4CooperativeMatrixINTEL"); add(CapabilityFunctionVariantsINTEL, "FunctionVariantsINTEL"); add(CapabilitySpecConditionalINTEL, "SpecConditionalINTEL"); add(internal::CapabilityBFloat16ArithmeticINTEL, "BFloat16ArithmeticINTEL"); } SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap) template <> inline void SPIRVMap::init() { add(InitializationModeQualifierInitOnDeviceReprogramINTEL, "InitOnDeviceReprogramINTEL"); add(InitializationModeQualifierInitOnDeviceResetINTEL, "InitOnDeviceResetINTEL"); add(InitializationModeQualifierMax, "Max"); } SPIRV_DEF_NAMEMAP(InitializationModeQualifier, SPIRVInitializationModeQualifierNameMap) template <> inline void SPIRVMap::init() { add(HostAccessQualifierNoneINTEL, "NoneINTEL"); add(HostAccessQualifierReadINTEL, "ReadINTEL"); add(HostAccessQualifierWriteINTEL, "WriteINTEL"); add(HostAccessQualifierReadWriteINTEL, "ReadWriteINTEL"); add(HostAccessQualifierMax, "Max"); } SPIRV_DEF_NAMEMAP(HostAccessQualifier, SPIRVHostAccessQualifierNameMap) template <> inline void SPIRVMap::init() { add(NamedMaximumNumberOfRegistersAutoINTEL, "AutoINTEL"); } SPIRV_DEF_NAMEMAP(NamedMaximumNumberOfRegisters, SPIRVNamedMaximumNumberOfRegistersNameMap); } /* namespace SPIRV */ #endif // SPIRV_LIBSPIRV_SPIRVNAMEMAPENUM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVOpCode.h000066400000000000000000000235741506646577400227370ustar00rootroot00000000000000//===- SPIRVOpCode.h - Class to represent SPIR-V Operation Codes -*- C++ -*-==// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Operation Code class for SPIR-V. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVOPCODE_H #define SPIRV_LIBSPIRV_SPIRVOPCODE_H #include "SPIRVUtil.h" #include "spirv/unified1/spirv.hpp" #include "spirv_internal.hpp" #include using namespace spv; namespace SPIRV { template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, ...) add(Op##x, #x); #define _SPIRV_OP_INTERNAL(x, ...) add(internal::Op##x, #x); #include "SPIRVOpCodeEnum.h" #include "SPIRVOpCodeEnumInternal.h" #undef _SPIRV_OP_INTERNAL #undef _SPIRV_OP } SPIRV_DEF_NAMEMAP(Op, OpCodeNameMap) inline bool isFPAtomicOpCode(Op OpCode) { return OpCode == OpAtomicFAddEXT || OpCode == OpAtomicFMinEXT || OpCode == OpAtomicFMaxEXT; } inline bool isAtomicOpCode(Op OpCode) { static_assert(OpAtomicLoad < OpAtomicXor, ""); return ((unsigned)OpCode >= OpAtomicLoad && (unsigned)OpCode <= OpAtomicXor) || OpCode == OpAtomicFlagTestAndSet || OpCode == OpAtomicFlagClear || isFPAtomicOpCode(OpCode); } inline bool isBinaryOpCode(Op OpCode) { return ((unsigned)OpCode >= OpIAdd && (unsigned)OpCode <= OpFMod) || OpCode == OpDot || OpCode == OpIAddCarry || OpCode == OpISubBorrow || OpCode == OpUMulExtended || OpCode == OpSMulExtended; } inline bool isBinaryPtrOpCode(Op OpCode) { return (unsigned)OpCode >= OpPtrEqual && (unsigned)OpCode <= OpPtrDiff; } inline bool isShiftOpCode(Op OpCode) { return (unsigned)OpCode >= OpShiftRightLogical && (unsigned)OpCode <= OpShiftLeftLogical; } inline bool isLogicalOpCode(Op OpCode) { return (unsigned)OpCode >= OpLogicalEqual && (unsigned)OpCode <= OpLogicalNot; } inline bool isUnaryPredicateOpCode(Op OpCode) { return (unsigned)OpCode >= OpAny && (unsigned)OpCode <= OpSignBitSet; } inline bool isBitwiseOpCode(Op OpCode) { return (unsigned)OpCode >= OpBitwiseOr && (unsigned)OpCode <= OpBitwiseAnd; } inline bool isBinaryShiftLogicalBitwiseOpCode(Op OpCode) { return (((unsigned)OpCode >= OpShiftRightLogical && (unsigned)OpCode <= OpBitwiseAnd) || isBinaryOpCode(OpCode)); } inline bool isCmpOpCode(Op OpCode) { return ((unsigned)OpCode >= OpIEqual && (unsigned)OpCode <= OpFUnordGreaterThanEqual) || (OpCode >= OpLessOrGreater && OpCode <= OpLogicalNotEqual); } inline bool isCvtOpCode(Op OpCode) { return ((unsigned)OpCode >= OpConvertFToU && (unsigned)OpCode <= OpBitcast) || OpCode == OpSatConvertSToU || OpCode == OpSatConvertUToS || OpCode == OpPtrCastToCrossWorkgroupINTEL || OpCode == OpCrossWorkgroupCastToPtrINTEL; } inline bool isCvtToUnsignedOpCode(Op OpCode) { return OpCode == OpConvertFToU || OpCode == OpUConvert || OpCode == OpSatConvertSToU; } inline bool isCvtFromUnsignedOpCode(Op OpCode) { return OpCode == OpConvertUToF || OpCode == OpUConvert || OpCode == OpSatConvertUToS; } inline bool isSatCvtOpCode(Op OpCode) { return OpCode == OpSatConvertUToS || OpCode == OpSatConvertSToU; } inline bool isOpaqueGenericTypeOpCode(Op OpCode) { return ((unsigned)OpCode >= OpTypeEvent && (unsigned)OpCode <= OpTypeQueue) || OpCode == OpTypeSampler; } inline bool isGenericNegateOpCode(Op OpCode) { return (unsigned)OpCode == OpSNegate || (unsigned)OpCode == OpFNegate || (unsigned)OpCode == OpNot; } inline bool isAccessChainOpCode(Op OpCode) { return OpCode == OpAccessChain || OpCode == OpInBoundsAccessChain; } inline bool hasExecScope(Op OpCode) { unsigned OC = OpCode; return (OpGroupWaitEvents <= OC && OC <= OpGroupSMax) || (OpGroupReserveReadPipePackets <= OC && OC <= OpGroupCommitWritePipe) || (OC == OpGroupNonUniformRotateKHR); } inline bool hasGroupOperation(Op OpCode) { unsigned OC = OpCode; return (OpGroupIAdd <= OC && OC <= OpGroupSMax) || (OpGroupNonUniformBallotBitCount == OC) || (OpGroupNonUniformIAdd <= OC && OC <= OpGroupNonUniformLogicalXor) || (OpGroupIMulKHR <= OC && OC <= OpGroupLogicalXorKHR); } inline bool isUniformArithmeticOpCode(Op OpCode) { unsigned OC = OpCode; return (OpGroupIAdd <= OC && OC <= OpGroupSMax) || (OpGroupIMulKHR <= OC && OC <= OpGroupLogicalXorKHR); } inline bool isNonUniformArithmeticOpCode(Op OpCode) { unsigned OC = OpCode; return (OpGroupNonUniformIAdd <= OC && OC <= OpGroupNonUniformLogicalXor); } inline bool isGroupLogicalOpCode(Op OpCode) { unsigned OC = OpCode; return OC == OpGroupNonUniformLogicalAnd || OC == OpGroupNonUniformLogicalOr || OC == OpGroupNonUniformLogicalXor || OC == OpGroupLogicalAndKHR || OC == OpGroupLogicalOrKHR || OC == OpGroupLogicalXorKHR; } inline bool isGroupOpCode(Op OpCode) { unsigned OC = OpCode; return (OpGroupAll <= OC && OC <= OpGroupSMax) || (OpGroupIMulKHR <= OC && OC <= OpGroupLogicalXorKHR); } inline bool isGroupNonUniformOpcode(Op OpCode) { unsigned OC = OpCode; return (OpGroupNonUniformElect <= OC && OC <= OpGroupNonUniformQuadSwap) || (OC == OpGroupNonUniformRotateKHR); } inline bool isMediaBlockINTELOpcode(Op OpCode) { return OpCode == OpSubgroupImageMediaBlockReadINTEL || OpCode == OpSubgroupImageMediaBlockWriteINTEL; } inline bool isPipeOpCode(Op OpCode) { unsigned OC = OpCode; return (OpReadPipe <= OC && OC <= OpGroupCommitWritePipe) || (OpReadPipeBlockingINTEL <= OC && OC <= OpWritePipeBlockingINTEL); } inline bool isSubgroupAvcINTELTypeOpCode(Op OpCode) { unsigned OC = OpCode; return OpTypeAvcImePayloadINTEL <= OC && OC <= OpTypeAvcSicResultINTEL; } inline bool isSubgroupAvcINTELInstructionOpCode(Op OpCode) { unsigned OC = OpCode; return OpSubgroupAvcMceGetDefaultInterBaseMultiReferencePenaltyINTEL <= OC && OC <= OpSubgroupAvcSicGetInterRawSadsINTEL; } inline bool isSubgroupAvcINTELEvaluateOpcode(Op OpCode) { unsigned OC = OpCode; return (OpSubgroupAvcImeEvaluateWithSingleReferenceINTEL <= OC && OC <= OpSubgroupAvcImeEvaluateWithDualReferenceStreaminoutINTEL) || (OpSubgroupAvcRefEvaluateWithSingleReferenceINTEL <= OC && OC <= OpSubgroupAvcRefEvaluateWithMultiReferenceInterlacedINTEL) || (OpSubgroupAvcSicEvaluateIpeINTEL <= OC && OC <= OpSubgroupAvcSicEvaluateWithMultiReferenceInterlacedINTEL); } inline bool isVCOpCode(Op OpCode) { return OpCode == OpTypeBufferSurfaceINTEL; } inline bool isTypeOpCode(Op OpCode) { unsigned OC = OpCode; return (OpTypeVoid <= OC && OC <= OpTypePipe) || OC == OpTypePipeStorage || isSubgroupAvcINTELTypeOpCode(OpCode) || OC == OpTypeVmeImageINTEL || isVCOpCode(OpCode) || OC == internal::OpTypeTokenINTEL || OC == internal::OpTypeJointMatrixINTEL || OC == OpTypeCooperativeMatrixKHR; } inline bool isFnVarSpecConstINTEL(Op OpCode) { const unsigned OC = OpCode; return OC == OpSpecConstantArchitectureINTEL || OC == OpSpecConstantTargetINTEL || OC == OpSpecConstantCapabilitiesINTEL; } inline bool isSpecConstantOpCode(Op OpCode) { unsigned OC = OpCode; return (OpSpecConstantTrue <= OC && OC <= OpSpecConstantOp) || isFnVarSpecConstINTEL(OpCode); } inline bool isConstantOpCode(Op OpCode) { unsigned OC = OpCode; return (OpConstantTrue <= OC && OC <= OpSpecConstantOp) || OC == OpUndef || OC == OpConstantPipeStorage || OC == OpConstantFunctionPointerINTEL || isSpecConstantOpCode(OpCode); } inline bool isModuleScopeAllowedOpCode(Op OpCode) { return OpCode == OpVariable || OpCode == OpExtInst || isConstantOpCode(OpCode); } inline bool isIntelSubgroupOpCode(Op OpCode) { unsigned OC = OpCode; return OpSubgroupShuffleINTEL <= OC && OC <= OpSubgroupImageBlockWriteINTEL; } inline bool isEventOpCode(Op OpCode) { return OpRetainEvent <= OpCode && OpCode <= OpCaptureEventProfilingInfo; } inline bool isSplitBarrierINTELOpCode(Op OpCode) { return OpCode == OpControlBarrierArriveINTEL || OpCode == OpControlBarrierWaitINTEL; } } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVOPCODE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h000066400000000000000000000540301506646577400235530ustar00rootroot00000000000000_SPIRV_OP(Nop, 0) _SPIRV_OP(Undef, 1) _SPIRV_OP(SourceContinued, 2) _SPIRV_OP(Source, 3) _SPIRV_OP(SourceExtension, 4) _SPIRV_OP(Name, 5) _SPIRV_OP(MemberName, 6) _SPIRV_OP(String, 7) _SPIRV_OP(Line, 8) _SPIRV_OP(Extension, 10) _SPIRV_OP(ExtInstImport, 11) _SPIRV_OP(ExtInst, 12) _SPIRV_OP(MemoryModel, 14) _SPIRV_OP(EntryPoint, 15) _SPIRV_OP(ExecutionMode, 16) _SPIRV_OP(Capability, 17) _SPIRV_OP(TypeVoid, 19) _SPIRV_OP(TypeBool, 20) _SPIRV_OP(TypeInt, 21) _SPIRV_OP(TypeFloat, 22) _SPIRV_OP(TypeVector, 23) _SPIRV_OP(TypeMatrix, 24) _SPIRV_OP(TypeImage, 25) _SPIRV_OP(TypeSampler, 26) _SPIRV_OP(TypeSampledImage, 27) _SPIRV_OP(TypeArray, 28) _SPIRV_OP(TypeRuntimeArray, 29) _SPIRV_OP(TypeStruct, 30) _SPIRV_OP(TypeOpaque, 31) _SPIRV_OP(TypePointer, 32) _SPIRV_OP(TypeFunction, 33) _SPIRV_OP(TypeEvent, 34) _SPIRV_OP(TypeDeviceEvent, 35) _SPIRV_OP(TypeReserveId, 36) _SPIRV_OP(TypeQueue, 37) _SPIRV_OP(TypePipe, 38) _SPIRV_OP(TypeForwardPointer, 39) _SPIRV_OP(ConstantTrue, 41) _SPIRV_OP(ConstantFalse, 42) _SPIRV_OP(Constant, 43) _SPIRV_OP(ConstantComposite, 44) _SPIRV_OP(ConstantSampler, 45) _SPIRV_OP(ConstantNull, 46) _SPIRV_OP(SpecConstantTrue, 48) _SPIRV_OP(SpecConstantFalse, 49) _SPIRV_OP(SpecConstant, 50) _SPIRV_OP(SpecConstantComposite, 51) _SPIRV_OP(SpecConstantOp, 52) _SPIRV_OP(Function, 54) _SPIRV_OP(FunctionParameter, 55) _SPIRV_OP(FunctionEnd, 56) _SPIRV_OP(FunctionCall, 57) _SPIRV_OP(Variable, 59) _SPIRV_OP(ImageTexelPointer, 60) _SPIRV_OP(Load, 61) _SPIRV_OP(Store, 62) _SPIRV_OP(CopyMemory, 63) _SPIRV_OP(CopyMemorySized, 64) _SPIRV_OP(AccessChain, 65) _SPIRV_OP(InBoundsAccessChain, 66) _SPIRV_OP(PtrAccessChain, 67) _SPIRV_OP(ArrayLength, 68) _SPIRV_OP(GenericPtrMemSemantics, 69) _SPIRV_OP(InBoundsPtrAccessChain, 70) _SPIRV_OP(Decorate, 71) _SPIRV_OP(MemberDecorate, 72) _SPIRV_OP(DecorationGroup, 73) _SPIRV_OP(GroupDecorate, 74) _SPIRV_OP(GroupMemberDecorate, 75) _SPIRV_OP(VectorExtractDynamic, 77) _SPIRV_OP(VectorInsertDynamic, 78) _SPIRV_OP(VectorShuffle, 79) _SPIRV_OP(CompositeConstruct, 80) _SPIRV_OP(CompositeExtract, 81) _SPIRV_OP(CompositeInsert, 82) _SPIRV_OP(CopyObject, 83) _SPIRV_OP(Transpose, 84) _SPIRV_OP(SampledImage, 86) _SPIRV_OP(ImageSampleImplicitLod, 87) _SPIRV_OP(ImageSampleExplicitLod, 88) _SPIRV_OP(ImageSampleDrefImplicitLod, 89) _SPIRV_OP(ImageSampleDrefExplicitLod, 90) _SPIRV_OP(ImageSampleProjImplicitLod, 91) _SPIRV_OP(ImageSampleProjExplicitLod, 92) _SPIRV_OP(ImageSampleProjDrefImplicitLod, 93) _SPIRV_OP(ImageSampleProjDrefExplicitLod, 94) _SPIRV_OP(ImageFetch, 95) _SPIRV_OP(ImageGather, 96) _SPIRV_OP(ImageDrefGather, 97) _SPIRV_OP(ImageRead, 98) _SPIRV_OP(ImageWrite, 99) _SPIRV_OP(Image, 100) _SPIRV_OP(ImageQueryFormat, 101) _SPIRV_OP(ImageQueryOrder, 102) _SPIRV_OP(ImageQuerySizeLod, 103) _SPIRV_OP(ImageQuerySize, 104) _SPIRV_OP(ImageQueryLod, 105) _SPIRV_OP(ImageQueryLevels, 106) _SPIRV_OP(ImageQuerySamples, 107) _SPIRV_OP(ConvertFToU, 109) _SPIRV_OP(ConvertFToS, 110) _SPIRV_OP(ConvertSToF, 111) _SPIRV_OP(ConvertUToF, 112) _SPIRV_OP(UConvert, 113) _SPIRV_OP(SConvert, 114) _SPIRV_OP(FConvert, 115) _SPIRV_OP(QuantizeToF16, 116) _SPIRV_OP(ConvertPtrToU, 117) _SPIRV_OP(SatConvertSToU, 118) _SPIRV_OP(SatConvertUToS, 119) _SPIRV_OP(ConvertUToPtr, 120) _SPIRV_OP(PtrCastToGeneric, 121) _SPIRV_OP(GenericCastToPtr, 122) _SPIRV_OP(GenericCastToPtrExplicit, 123) _SPIRV_OP(Bitcast, 124) _SPIRV_OP(SNegate, 126) _SPIRV_OP(FNegate, 127) _SPIRV_OP(IAdd, 128) _SPIRV_OP(FAdd, 129) _SPIRV_OP(ISub, 130) _SPIRV_OP(FSub, 131) _SPIRV_OP(IMul, 132) _SPIRV_OP(FMul, 133) _SPIRV_OP(UDiv, 134) _SPIRV_OP(SDiv, 135) _SPIRV_OP(FDiv, 136) _SPIRV_OP(UMod, 137) _SPIRV_OP(SRem, 138) _SPIRV_OP(SMod, 139) _SPIRV_OP(FRem, 140) _SPIRV_OP(FMod, 141) _SPIRV_OP(VectorTimesScalar, 142) _SPIRV_OP(MatrixTimesScalar, 143) _SPIRV_OP(VectorTimesMatrix, 144) _SPIRV_OP(MatrixTimesVector, 145) _SPIRV_OP(MatrixTimesMatrix, 146) _SPIRV_OP(OuterProduct, 147) _SPIRV_OP(Dot, 148) _SPIRV_OP(IAddCarry, 149) _SPIRV_OP(ISubBorrow, 150) _SPIRV_OP(UMulExtended, 151) _SPIRV_OP(SMulExtended, 152) _SPIRV_OP(Any, 154) _SPIRV_OP(All, 155) _SPIRV_OP(IsNan, 156) _SPIRV_OP(IsInf, 157) _SPIRV_OP(IsFinite, 158) _SPIRV_OP(IsNormal, 159) _SPIRV_OP(SignBitSet, 160) _SPIRV_OP(LessOrGreater, 161) _SPIRV_OP(Ordered, 162) _SPIRV_OP(Unordered, 163) _SPIRV_OP(LogicalEqual, 164) _SPIRV_OP(LogicalNotEqual, 165) _SPIRV_OP(LogicalOr, 166) _SPIRV_OP(LogicalAnd, 167) _SPIRV_OP(LogicalNot, 168) _SPIRV_OP(Select, 169) _SPIRV_OP(IEqual, 170) _SPIRV_OP(INotEqual, 171) _SPIRV_OP(UGreaterThan, 172) _SPIRV_OP(SGreaterThan, 173) _SPIRV_OP(UGreaterThanEqual, 174) _SPIRV_OP(SGreaterThanEqual, 175) _SPIRV_OP(ULessThan, 176) _SPIRV_OP(SLessThan, 177) _SPIRV_OP(ULessThanEqual, 178) _SPIRV_OP(SLessThanEqual, 179) _SPIRV_OP(FOrdEqual, 180) _SPIRV_OP(FUnordEqual, 181) _SPIRV_OP(FOrdNotEqual, 182) _SPIRV_OP(FUnordNotEqual, 183) _SPIRV_OP(FOrdLessThan, 184) _SPIRV_OP(FUnordLessThan, 185) _SPIRV_OP(FOrdGreaterThan, 186) _SPIRV_OP(FUnordGreaterThan, 187) _SPIRV_OP(FOrdLessThanEqual, 188) _SPIRV_OP(FUnordLessThanEqual, 189) _SPIRV_OP(FOrdGreaterThanEqual, 190) _SPIRV_OP(FUnordGreaterThanEqual, 191) _SPIRV_OP(ShiftRightLogical, 194) _SPIRV_OP(ShiftRightArithmetic, 195) _SPIRV_OP(ShiftLeftLogical, 196) _SPIRV_OP(BitwiseOr, 197) _SPIRV_OP(BitwiseXor, 198) _SPIRV_OP(BitwiseAnd, 199) _SPIRV_OP(Not, 200) _SPIRV_OP(BitFieldInsert, 201) _SPIRV_OP(BitFieldSExtract, 202) _SPIRV_OP(BitFieldUExtract, 203) _SPIRV_OP(BitReverse, 204) _SPIRV_OP(BitCount, 205) _SPIRV_OP(DPdx, 207) _SPIRV_OP(DPdy, 208) _SPIRV_OP(Fwidth, 209) _SPIRV_OP(DPdxFine, 210) _SPIRV_OP(DPdyFine, 211) _SPIRV_OP(FwidthFine, 212) _SPIRV_OP(DPdxCoarse, 213) _SPIRV_OP(DPdyCoarse, 214) _SPIRV_OP(FwidthCoarse, 215) _SPIRV_OP(EmitVertex, 218) _SPIRV_OP(EndPrimitive, 219) _SPIRV_OP(EmitStreamVertex, 220) _SPIRV_OP(EndStreamPrimitive, 221) _SPIRV_OP(ControlBarrier, 224) _SPIRV_OP(MemoryBarrier, 225) _SPIRV_OP(AtomicLoad, 227) _SPIRV_OP(AtomicStore, 228) _SPIRV_OP(AtomicExchange, 229) _SPIRV_OP(AtomicCompareExchange, 230) _SPIRV_OP(AtomicCompareExchangeWeak, 231) _SPIRV_OP(AtomicIIncrement, 232) _SPIRV_OP(AtomicIDecrement, 233) _SPIRV_OP(AtomicIAdd, 234) _SPIRV_OP(AtomicISub, 235) _SPIRV_OP(AtomicSMin, 236) _SPIRV_OP(AtomicUMin, 237) _SPIRV_OP(AtomicSMax, 238) _SPIRV_OP(AtomicUMax, 239) _SPIRV_OP(AtomicAnd, 240) _SPIRV_OP(AtomicOr, 241) _SPIRV_OP(AtomicXor, 242) _SPIRV_OP(Phi, 245) _SPIRV_OP(LoopMerge, 246) _SPIRV_OP(SelectionMerge, 247) _SPIRV_OP(Label, 248) _SPIRV_OP(Branch, 249) _SPIRV_OP(BranchConditional, 250) _SPIRV_OP(Switch, 251) _SPIRV_OP(Kill, 252) _SPIRV_OP(Return, 253) _SPIRV_OP(ReturnValue, 254) _SPIRV_OP(Unreachable, 255) _SPIRV_OP(LifetimeStart, 256) _SPIRV_OP(LifetimeStop, 257) _SPIRV_OP(GroupAsyncCopy, 259) _SPIRV_OP(GroupWaitEvents, 260) _SPIRV_OP(GroupAll, 261) _SPIRV_OP(GroupAny, 262) _SPIRV_OP(GroupBroadcast, 263) _SPIRV_OP(GroupIAdd, 264) _SPIRV_OP(GroupFAdd, 265) _SPIRV_OP(GroupFMin, 266) _SPIRV_OP(GroupUMin, 267) _SPIRV_OP(GroupSMin, 268) _SPIRV_OP(GroupFMax, 269) _SPIRV_OP(GroupUMax, 270) _SPIRV_OP(GroupSMax, 271) _SPIRV_OP(ReadPipe, 274) _SPIRV_OP(WritePipe, 275) _SPIRV_OP(ReservedReadPipe, 276) _SPIRV_OP(ReservedWritePipe, 277) _SPIRV_OP(ReserveReadPipePackets, 278) _SPIRV_OP(ReserveWritePipePackets, 279) _SPIRV_OP(CommitReadPipe, 280) _SPIRV_OP(CommitWritePipe, 281) _SPIRV_OP(IsValidReserveId, 282) _SPIRV_OP(GetNumPipePackets, 283) _SPIRV_OP(GetMaxPipePackets, 284) _SPIRV_OP(GroupReserveReadPipePackets, 285) _SPIRV_OP(GroupReserveWritePipePackets, 286) _SPIRV_OP(GroupCommitReadPipe, 287) _SPIRV_OP(GroupCommitWritePipe, 288) _SPIRV_OP(EnqueueMarker, 291) _SPIRV_OP(EnqueueKernel, 292) _SPIRV_OP(GetKernelNDrangeSubGroupCount, 293) _SPIRV_OP(GetKernelNDrangeMaxSubGroupSize, 294) _SPIRV_OP(GetKernelWorkGroupSize, 295) _SPIRV_OP(GetKernelPreferredWorkGroupSizeMultiple, 296) _SPIRV_OP(RetainEvent, 297) _SPIRV_OP(ReleaseEvent, 298) _SPIRV_OP(CreateUserEvent, 299) _SPIRV_OP(IsValidEvent, 300) _SPIRV_OP(SetUserEventStatus, 301) _SPIRV_OP(CaptureEventProfilingInfo, 302) _SPIRV_OP(GetDefaultQueue, 303) _SPIRV_OP(BuildNDRange, 304) _SPIRV_OP(ImageSparseSampleImplicitLod, 305) _SPIRV_OP(ImageSparseSampleExplicitLod, 306) _SPIRV_OP(ImageSparseSampleDrefImplicitLod, 307) _SPIRV_OP(ImageSparseSampleDrefExplicitLod, 308) _SPIRV_OP(ImageSparseSampleProjImplicitLod, 309) _SPIRV_OP(ImageSparseSampleProjExplicitLod, 310) _SPIRV_OP(ImageSparseSampleProjDrefImplicitLod, 311) _SPIRV_OP(ImageSparseSampleProjDrefExplicitLod, 312) _SPIRV_OP(ImageSparseFetch, 313) _SPIRV_OP(ImageSparseGather, 314) _SPIRV_OP(ImageSparseDrefGather, 315) _SPIRV_OP(ImageSparseTexelsResident, 316) _SPIRV_OP(NoLine, 317) _SPIRV_OP(AtomicFlagTestAndSet, 318) _SPIRV_OP(AtomicFlagClear, 319) _SPIRV_OP(SizeOf, 321) _SPIRV_OP(TypePipeStorage, 322) _SPIRV_OP(ConstantPipeStorage, 323) _SPIRV_OP(CreatePipeFromPipeStorage, 324) _SPIRV_OP(ModuleProcessed, 330) _SPIRV_OP(ExecutionModeId, 331) _SPIRV_OP(DecorateId, 332) _SPIRV_OP(GroupNonUniformElect, 333) _SPIRV_OP(GroupNonUniformAll, 334) _SPIRV_OP(GroupNonUniformAny, 335) _SPIRV_OP(GroupNonUniformAllEqual, 336) _SPIRV_OP(GroupNonUniformBroadcast, 337) _SPIRV_OP(GroupNonUniformBroadcastFirst, 338) _SPIRV_OP(GroupNonUniformBallot, 339) _SPIRV_OP(GroupNonUniformInverseBallot, 340) _SPIRV_OP(GroupNonUniformBallotBitExtract, 341) _SPIRV_OP(GroupNonUniformBallotBitCount, 342) _SPIRV_OP(GroupNonUniformBallotFindLSB, 343) _SPIRV_OP(GroupNonUniformBallotFindMSB, 344) _SPIRV_OP(GroupNonUniformShuffle, 345) _SPIRV_OP(GroupNonUniformShuffleXor, 346) _SPIRV_OP(GroupNonUniformShuffleUp, 347) _SPIRV_OP(GroupNonUniformShuffleDown, 348) _SPIRV_OP(GroupNonUniformIAdd, 349) _SPIRV_OP(GroupNonUniformFAdd, 350) _SPIRV_OP(GroupNonUniformIMul, 351) _SPIRV_OP(GroupNonUniformFMul, 352) _SPIRV_OP(GroupNonUniformSMin, 353) _SPIRV_OP(GroupNonUniformUMin, 354) _SPIRV_OP(GroupNonUniformFMin, 355) _SPIRV_OP(GroupNonUniformSMax, 356) _SPIRV_OP(GroupNonUniformUMax, 357) _SPIRV_OP(GroupNonUniformFMax, 358) _SPIRV_OP(GroupNonUniformBitwiseAnd, 359) _SPIRV_OP(GroupNonUniformBitwiseOr, 360) _SPIRV_OP(GroupNonUniformBitwiseXor, 361) _SPIRV_OP(GroupNonUniformLogicalAnd, 362) _SPIRV_OP(GroupNonUniformLogicalOr, 363) _SPIRV_OP(GroupNonUniformLogicalXor, 364) _SPIRV_OP(CopyLogical, 400) _SPIRV_OP(PtrEqual, 401) _SPIRV_OP(PtrNotEqual, 402) _SPIRV_OP(PtrDiff, 403) _SPIRV_OP(GroupNonUniformRotateKHR, 4431) _SPIRV_OP(SDotKHR, 4450) _SPIRV_OP(UDotKHR, 4451) _SPIRV_OP(SUDotKHR, 4452) _SPIRV_OP(SDotAccSatKHR, 4453) _SPIRV_OP(UDotAccSatKHR, 4454) _SPIRV_OP(SUDotAccSatKHR, 4455) _SPIRV_OP(ReadClockKHR, 5056) _SPIRV_OP(TypeCooperativeMatrixKHR, 4456) _SPIRV_OP(CooperativeMatrixLoadKHR, 4457) _SPIRV_OP(CooperativeMatrixStoreKHR, 4458) _SPIRV_OP(CooperativeMatrixMulAddKHR, 4459) _SPIRV_OP(CooperativeMatrixLengthKHR, 4460) _SPIRV_OP(SubgroupShuffleINTEL, 5571) _SPIRV_OP(SubgroupShuffleDownINTEL, 5572) _SPIRV_OP(SubgroupShuffleUpINTEL, 5573) _SPIRV_OP(SubgroupShuffleXorINTEL, 5574) _SPIRV_OP(SubgroupBlockReadINTEL, 5575) _SPIRV_OP(SubgroupBlockWriteINTEL, 5576) _SPIRV_OP(SubgroupImageBlockReadINTEL, 5577) _SPIRV_OP(SubgroupImageBlockWriteINTEL, 5578) _SPIRV_OP(SubgroupImageMediaBlockReadINTEL, 5580) _SPIRV_OP(SubgroupImageMediaBlockWriteINTEL, 5581) _SPIRV_OP(ConstantFunctionPointerINTEL, 5600) _SPIRV_OP(FunctionPointerCallINTEL, 5601) _SPIRV_OP(AsmTargetINTEL, 5609) _SPIRV_OP(AsmINTEL, 5610) _SPIRV_OP(AsmCallINTEL, 5611) _SPIRV_OP(AtomicFMinEXT, 5614) _SPIRV_OP(AtomicFMaxEXT, 5615) _SPIRV_OP(AssumeTrueKHR, 5630) _SPIRV_OP(ExpectKHR, 5631) _SPIRV_OP(DecorateString, 5632) _SPIRV_OP(MemberDecorateString, 5633) _SPIRV_OP(VmeImageINTEL, 5699) _SPIRV_OP(TypeVmeImageINTEL, 5700) _SPIRV_OP(TypeAvcImePayloadINTEL, 5701) _SPIRV_OP(TypeAvcRefPayloadINTEL, 5702) _SPIRV_OP(TypeAvcSicPayloadINTEL, 5703) _SPIRV_OP(TypeAvcMcePayloadINTEL, 5704) _SPIRV_OP(TypeAvcMceResultINTEL, 5705) _SPIRV_OP(TypeAvcImeResultINTEL, 5706) _SPIRV_OP(TypeAvcImeResultSingleReferenceStreamoutINTEL, 5707) _SPIRV_OP(TypeAvcImeResultDualReferenceStreamoutINTEL, 5708) _SPIRV_OP(TypeAvcImeSingleReferenceStreaminINTEL, 5709) _SPIRV_OP(TypeAvcImeDualReferenceStreaminINTEL, 5710) _SPIRV_OP(TypeAvcRefResultINTEL, 5711) _SPIRV_OP(TypeAvcSicResultINTEL, 5712) _SPIRV_OP(SubgroupAvcMceGetDefaultInterBaseMultiReferencePenaltyINTEL, 5713) _SPIRV_OP(SubgroupAvcMceSetInterBaseMultiReferencePenaltyINTEL, 5714) _SPIRV_OP(SubgroupAvcMceGetDefaultInterShapePenaltyINTEL, 5715) _SPIRV_OP(SubgroupAvcMceSetInterShapePenaltyINTEL, 5716) _SPIRV_OP(SubgroupAvcMceGetDefaultInterDirectionPenaltyINTEL, 5717) _SPIRV_OP(SubgroupAvcMceSetInterDirectionPenaltyINTEL, 5718) _SPIRV_OP(SubgroupAvcMceGetDefaultIntraLumaShapePenaltyINTEL, 5719) _SPIRV_OP(SubgroupAvcMceGetDefaultInterMotionVectorCostTableINTEL, 5720) _SPIRV_OP(SubgroupAvcMceGetDefaultHighPenaltyCostTableINTEL, 5721) _SPIRV_OP(SubgroupAvcMceGetDefaultMediumPenaltyCostTableINTEL, 5722) _SPIRV_OP(SubgroupAvcMceGetDefaultLowPenaltyCostTableINTEL, 5723) _SPIRV_OP(SubgroupAvcMceSetMotionVectorCostFunctionINTEL, 5724) _SPIRV_OP(SubgroupAvcMceGetDefaultIntraLumaModePenaltyINTEL, 5725) _SPIRV_OP(SubgroupAvcMceGetDefaultNonDcLumaIntraPenaltyINTEL, 5726) _SPIRV_OP(SubgroupAvcMceGetDefaultIntraChromaModeBasePenaltyINTEL, 5727) _SPIRV_OP(SubgroupAvcMceSetAcOnlyHaarINTEL, 5728) _SPIRV_OP(SubgroupAvcMceSetSourceInterlacedFieldPolarityINTEL, 5729) _SPIRV_OP(SubgroupAvcMceSetSingleReferenceInterlacedFieldPolarityINTEL, 5730) _SPIRV_OP(SubgroupAvcMceSetDualReferenceInterlacedFieldPolaritiesINTEL, 5731) _SPIRV_OP(SubgroupAvcMceConvertToImePayloadINTEL, 5732) _SPIRV_OP(SubgroupAvcMceConvertToImeResultINTEL, 5733) _SPIRV_OP(SubgroupAvcMceConvertToRefPayloadINTEL, 5734) _SPIRV_OP(SubgroupAvcMceConvertToRefResultINTEL, 5735) _SPIRV_OP(SubgroupAvcMceConvertToSicPayloadINTEL, 5736) _SPIRV_OP(SubgroupAvcMceConvertToSicResultINTEL, 5737) _SPIRV_OP(SubgroupAvcMceGetMotionVectorsINTEL, 5738) _SPIRV_OP(SubgroupAvcMceGetInterDistortionsINTEL, 5739) _SPIRV_OP(SubgroupAvcMceGetBestInterDistortionsINTEL, 5740) _SPIRV_OP(SubgroupAvcMceGetInterMajorShapeINTEL, 5741) _SPIRV_OP(SubgroupAvcMceGetInterMinorShapeINTEL, 5742) _SPIRV_OP(SubgroupAvcMceGetInterDirectionsINTEL, 5743) _SPIRV_OP(SubgroupAvcMceGetInterMotionVectorCountINTEL, 5744) _SPIRV_OP(SubgroupAvcMceGetInterReferenceIdsINTEL, 5745) _SPIRV_OP(SubgroupAvcMceGetInterReferenceInterlacedFieldPolaritiesINTEL, 5746) _SPIRV_OP(SubgroupAvcImeInitializeINTEL, 5747) _SPIRV_OP(SubgroupAvcImeSetSingleReferenceINTEL, 5748) _SPIRV_OP(SubgroupAvcImeSetDualReferenceINTEL, 5749) _SPIRV_OP(SubgroupAvcImeRefWindowSizeINTEL, 5750) _SPIRV_OP(SubgroupAvcImeAdjustRefOffsetINTEL, 5751) _SPIRV_OP(SubgroupAvcImeConvertToMcePayloadINTEL, 5752) _SPIRV_OP(SubgroupAvcImeSetMaxMotionVectorCountINTEL, 5753) _SPIRV_OP(SubgroupAvcImeSetUnidirectionalMixDisableINTEL, 5754) _SPIRV_OP(SubgroupAvcImeSetEarlySearchTerminationThresholdINTEL, 5755) _SPIRV_OP(SubgroupAvcImeSetWeightedSadINTEL, 5756) _SPIRV_OP(SubgroupAvcImeEvaluateWithSingleReferenceINTEL, 5757) _SPIRV_OP(SubgroupAvcImeEvaluateWithDualReferenceINTEL, 5758) _SPIRV_OP(SubgroupAvcImeEvaluateWithSingleReferenceStreaminINTEL, 5759) _SPIRV_OP(SubgroupAvcImeEvaluateWithDualReferenceStreaminINTEL, 5760) _SPIRV_OP(SubgroupAvcImeEvaluateWithSingleReferenceStreamoutINTEL, 5761) _SPIRV_OP(SubgroupAvcImeEvaluateWithDualReferenceStreamoutINTEL, 5762) _SPIRV_OP(SubgroupAvcImeEvaluateWithSingleReferenceStreaminoutINTEL, 5763) _SPIRV_OP(SubgroupAvcImeEvaluateWithDualReferenceStreaminoutINTEL, 5764) _SPIRV_OP(SubgroupAvcImeConvertToMceResultINTEL, 5765) _SPIRV_OP(SubgroupAvcImeGetSingleReferenceStreaminINTEL, 5766) _SPIRV_OP(SubgroupAvcImeGetDualReferenceStreaminINTEL, 5767) _SPIRV_OP(SubgroupAvcImeStripSingleReferenceStreamoutINTEL, 5768) _SPIRV_OP(SubgroupAvcImeStripDualReferenceStreamoutINTEL, 5769) _SPIRV_OP(SubgroupAvcImeGetStreamoutSingleReferenceMajorShapeMotionVectorsINTEL, 5770) _SPIRV_OP(SubgroupAvcImeGetStreamoutSingleReferenceMajorShapeDistortionsINTEL, 5771) _SPIRV_OP(SubgroupAvcImeGetStreamoutSingleReferenceMajorShapeReferenceIdsINTEL, 5772) _SPIRV_OP(SubgroupAvcImeGetStreamoutDualReferenceMajorShapeMotionVectorsINTEL, 5773) _SPIRV_OP(SubgroupAvcImeGetStreamoutDualReferenceMajorShapeDistortionsINTEL, 5774) _SPIRV_OP(SubgroupAvcImeGetStreamoutDualReferenceMajorShapeReferenceIdsINTEL, 5775) _SPIRV_OP(SubgroupAvcImeGetBorderReachedINTEL, 5776) _SPIRV_OP(SubgroupAvcImeGetTruncatedSearchIndicationINTEL, 5777) _SPIRV_OP(SubgroupAvcImeGetUnidirectionalEarlySearchTerminationINTEL, 5778) _SPIRV_OP(SubgroupAvcImeGetWeightingPatternMinimumMotionVectorINTEL, 5779) _SPIRV_OP(SubgroupAvcImeGetWeightingPatternMinimumDistortionINTEL, 5780) _SPIRV_OP(SubgroupAvcFmeInitializeINTEL, 5781) _SPIRV_OP(SubgroupAvcBmeInitializeINTEL, 5782) _SPIRV_OP(SubgroupAvcRefConvertToMcePayloadINTEL, 5783) _SPIRV_OP(SubgroupAvcRefSetBidirectionalMixDisableINTEL, 5784) _SPIRV_OP(SubgroupAvcRefSetBilinearFilterEnableINTEL, 5785) _SPIRV_OP(SubgroupAvcRefEvaluateWithSingleReferenceINTEL, 5786) _SPIRV_OP(SubgroupAvcRefEvaluateWithDualReferenceINTEL, 5787) _SPIRV_OP(SubgroupAvcRefEvaluateWithMultiReferenceINTEL, 5788) _SPIRV_OP(SubgroupAvcRefEvaluateWithMultiReferenceInterlacedINTEL, 5789) _SPIRV_OP(SubgroupAvcRefConvertToMceResultINTEL, 5790) _SPIRV_OP(SubgroupAvcSicInitializeINTEL, 5791) _SPIRV_OP(SubgroupAvcSicConfigureSkcINTEL, 5792) _SPIRV_OP(SubgroupAvcSicConfigureIpeLumaINTEL, 5793) _SPIRV_OP(SubgroupAvcSicConfigureIpeLumaChromaINTEL, 5794) _SPIRV_OP(SubgroupAvcSicGetMotionVectorMaskINTEL, 5795) _SPIRV_OP(SubgroupAvcSicConvertToMcePayloadINTEL, 5796) _SPIRV_OP(SubgroupAvcSicSetIntraLumaShapePenaltyINTEL, 5797) _SPIRV_OP(SubgroupAvcSicSetIntraLumaModeCostFunctionINTEL, 5798) _SPIRV_OP(SubgroupAvcSicSetIntraChromaModeCostFunctionINTEL, 5799) _SPIRV_OP(SubgroupAvcSicSetBilinearFilterEnableINTEL, 5800) _SPIRV_OP(SubgroupAvcSicSetSkcForwardTransformEnableINTEL, 5801) _SPIRV_OP(SubgroupAvcSicSetBlockBasedRawSkipSadINTEL, 5802) _SPIRV_OP(SubgroupAvcSicEvaluateIpeINTEL, 5803) _SPIRV_OP(SubgroupAvcSicEvaluateWithSingleReferenceINTEL, 5804) _SPIRV_OP(SubgroupAvcSicEvaluateWithDualReferenceINTEL, 5805) _SPIRV_OP(SubgroupAvcSicEvaluateWithMultiReferenceINTEL, 5806) _SPIRV_OP(SubgroupAvcSicEvaluateWithMultiReferenceInterlacedINTEL, 5807) _SPIRV_OP(SubgroupAvcSicConvertToMceResultINTEL, 5808) _SPIRV_OP(SubgroupAvcSicGetIpeLumaShapeINTEL, 5809) _SPIRV_OP(SubgroupAvcSicGetBestIpeLumaDistortionINTEL, 5810) _SPIRV_OP(SubgroupAvcSicGetBestIpeChromaDistortionINTEL, 5811) _SPIRV_OP(SubgroupAvcSicGetPackedIpeLumaModesINTEL, 5812) _SPIRV_OP(SubgroupAvcSicGetIpeChromaModeINTEL, 5813) _SPIRV_OP(SubgroupAvcSicGetPackedSkcLumaCountThresholdINTEL, 5814) _SPIRV_OP(SubgroupAvcSicGetPackedSkcLumaSumThresholdINTEL, 5815) _SPIRV_OP(SubgroupAvcSicGetInterRawSadsINTEL, 5816) _SPIRV_OP(VariableLengthArrayINTEL, 5818) _SPIRV_OP(SaveMemoryINTEL, 5819) _SPIRV_OP(RestoreMemoryINTEL, 5820) _SPIRV_OP(ArbitraryFloatSinCosPiINTEL, 5840) _SPIRV_OP(ArbitraryFloatCastINTEL, 5841) _SPIRV_OP(ArbitraryFloatCastFromIntINTEL, 5842) _SPIRV_OP(ArbitraryFloatCastToIntINTEL, 5843) _SPIRV_OP(ArbitraryFloatAddINTEL, 5846) _SPIRV_OP(ArbitraryFloatSubINTEL, 5847) _SPIRV_OP(ArbitraryFloatMulINTEL, 5848) _SPIRV_OP(ArbitraryFloatDivINTEL, 5849) _SPIRV_OP(ArbitraryFloatGTINTEL, 5850) _SPIRV_OP(ArbitraryFloatGEINTEL, 5851) _SPIRV_OP(ArbitraryFloatLTINTEL, 5852) _SPIRV_OP(ArbitraryFloatLEINTEL, 5853) _SPIRV_OP(ArbitraryFloatEQINTEL, 5854) _SPIRV_OP(ArbitraryFloatRecipINTEL, 5855) _SPIRV_OP(ArbitraryFloatRSqrtINTEL, 5856) _SPIRV_OP(ArbitraryFloatCbrtINTEL, 5857) _SPIRV_OP(ArbitraryFloatHypotINTEL, 5858) _SPIRV_OP(ArbitraryFloatSqrtINTEL, 5859) _SPIRV_OP(ArbitraryFloatLogINTEL, 5860) _SPIRV_OP(ArbitraryFloatLog2INTEL, 5861) _SPIRV_OP(ArbitraryFloatLog10INTEL, 5862) _SPIRV_OP(ArbitraryFloatLog1pINTEL, 5863) _SPIRV_OP(ArbitraryFloatExpINTEL, 5864) _SPIRV_OP(ArbitraryFloatExp2INTEL, 5865) _SPIRV_OP(ArbitraryFloatExp10INTEL, 5866) _SPIRV_OP(ArbitraryFloatExpm1INTEL, 5867) _SPIRV_OP(ArbitraryFloatSinINTEL, 5868) _SPIRV_OP(ArbitraryFloatCosINTEL, 5869) _SPIRV_OP(ArbitraryFloatSinCosINTEL, 5870) _SPIRV_OP(ArbitraryFloatSinPiINTEL, 5871) _SPIRV_OP(ArbitraryFloatCosPiINTEL, 5872) _SPIRV_OP(ArbitraryFloatASinINTEL, 5873) _SPIRV_OP(ArbitraryFloatASinPiINTEL, 5874) _SPIRV_OP(ArbitraryFloatACosINTEL, 5875) _SPIRV_OP(ArbitraryFloatACosPiINTEL, 5876) _SPIRV_OP(ArbitraryFloatATanINTEL, 5877) _SPIRV_OP(ArbitraryFloatATanPiINTEL, 5878) _SPIRV_OP(ArbitraryFloatATan2INTEL, 5879) _SPIRV_OP(ArbitraryFloatPowINTEL, 5880) _SPIRV_OP(ArbitraryFloatPowRINTEL, 5881) _SPIRV_OP(ArbitraryFloatPowNINTEL, 5882) _SPIRV_OP(LoopControlINTEL, 5887) _SPIRV_OP(AliasDomainDeclINTEL, 5911) _SPIRV_OP(AliasScopeDeclINTEL, 5912) _SPIRV_OP(AliasScopeListDeclINTEL, 5913) _SPIRV_OP(FixedSqrtINTEL, 5923) _SPIRV_OP(FixedRecipINTEL, 5924) _SPIRV_OP(FixedRsqrtINTEL, 5925) _SPIRV_OP(FixedSinINTEL, 5926) _SPIRV_OP(FixedCosINTEL, 5927) _SPIRV_OP(FixedSinCosINTEL, 5928) _SPIRV_OP(FixedSinPiINTEL, 5929) _SPIRV_OP(FixedCosPiINTEL, 5930) _SPIRV_OP(FixedSinCosPiINTEL, 5931) _SPIRV_OP(FixedLogINTEL, 5932) _SPIRV_OP(FixedExpINTEL, 5933) _SPIRV_OP(PtrCastToCrossWorkgroupINTEL, 5934) _SPIRV_OP(CrossWorkgroupCastToPtrINTEL, 5938) _SPIRV_OP(ReadPipeBlockingINTEL, 5946) _SPIRV_OP(WritePipeBlockingINTEL, 5947) _SPIRV_OP(FPGARegINTEL, 5949) _SPIRV_OP(AtomicFAddEXT, 6035) _SPIRV_OP(TypeBufferSurfaceINTEL, 6086) _SPIRV_OP(TypeStructContinuedINTEL, 6090) _SPIRV_OP(ConstantCompositeContinuedINTEL, 6091) _SPIRV_OP(SpecConstantCompositeContinuedINTEL, 6092) _SPIRV_OP(ControlBarrierArriveINTEL, 6142) _SPIRV_OP(ControlBarrierWaitINTEL, 6143) _SPIRV_OP(ArithmeticFenceEXT, 6145) _SPIRV_OP(Subgroup2DBlockLoadINTEL, 6231) _SPIRV_OP(Subgroup2DBlockLoadTransformINTEL, 6232) _SPIRV_OP(Subgroup2DBlockLoadTransposeINTEL, 6233) _SPIRV_OP(Subgroup2DBlockPrefetchINTEL, 6234) _SPIRV_OP(Subgroup2DBlockStoreINTEL, 6235) _SPIRV_OP(SubgroupMatrixMultiplyAccumulateINTEL, 6237) _SPIRV_OP(BitwiseFunctionINTEL, 6242) _SPIRV_OP(ConditionalExtensionINTEL, 6248) _SPIRV_OP(ConditionalEntryPointINTEL, 6249) _SPIRV_OP(ConditionalCapabilityINTEL, 6250) _SPIRV_OP(SpecConstantTargetINTEL, 6251) _SPIRV_OP(SpecConstantArchitectureINTEL, 6252) _SPIRV_OP(SpecConstantCapabilitiesINTEL, 6253) _SPIRV_OP(ConditionalCopyObjectINTEL, 6254) _SPIRV_OP(GroupIMulKHR, 6401) _SPIRV_OP(GroupFMulKHR, 6402) _SPIRV_OP(GroupBitwiseAndKHR, 6403) _SPIRV_OP(GroupBitwiseOrKHR, 6404) _SPIRV_OP(GroupBitwiseXorKHR, 6405) _SPIRV_OP(GroupLogicalAndKHR, 6406) _SPIRV_OP(GroupLogicalOrKHR, 6407) _SPIRV_OP(GroupLogicalXorKHR, 6408) SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVOpCodeEnumInternal.h000066400000000000000000000042251506646577400252510ustar00rootroot00000000000000#include "spirv_internal.hpp" _SPIRV_OP_INTERNAL(Forward, internal::OpForward) _SPIRV_OP_INTERNAL(TypeTokenINTEL, internal::OpTypeTokenINTEL) _SPIRV_OP_INTERNAL(ConvertFToBF16INTEL, internal::OpConvertFToBF16INTEL) _SPIRV_OP_INTERNAL(ConvertBF16ToFINTEL, internal::OpConvertBF16ToFINTEL) _SPIRV_OP_INTERNAL(TypeJointMatrixINTEL, internal::OpTypeJointMatrixINTEL) _SPIRV_OP_INTERNAL(JointMatrixLoadINTEL, internal::OpJointMatrixLoadINTEL) _SPIRV_OP_INTERNAL(JointMatrixStoreINTEL, internal::OpJointMatrixStoreINTEL) _SPIRV_OP_INTERNAL(JointMatrixMadINTEL, internal::OpJointMatrixMadINTEL) _SPIRV_OP_INTERNAL(JointMatrixSUMadINTEL, internal::OpJointMatrixSUMadINTEL) _SPIRV_OP_INTERNAL(JointMatrixUSMadINTEL, internal::OpJointMatrixUSMadINTEL) _SPIRV_OP_INTERNAL(JointMatrixUUMadINTEL, internal::OpJointMatrixUUMadINTEL) _SPIRV_OP_INTERNAL(JointMatrixWorkItemLengthINTEL, internal::OpJointMatrixWorkItemLengthINTEL) _SPIRV_OP_INTERNAL(JointMatrixGetElementCoordINTEL, internal::OpJointMatrixGetElementCoordINTEL) _SPIRV_OP_INTERNAL(CooperativeMatrixPrefetchINTEL, internal::OpCooperativeMatrixPrefetchINTEL) _SPIRV_OP_INTERNAL(CooperativeMatrixLoadCheckedINTEL, internal::OpCooperativeMatrixLoadCheckedINTEL) _SPIRV_OP_INTERNAL(CooperativeMatrixStoreCheckedINTEL, internal::OpCooperativeMatrixStoreCheckedINTEL) _SPIRV_OP_INTERNAL(CooperativeMatrixConstructCheckedINTEL, internal::OpCooperativeMatrixConstructCheckedINTEL) _SPIRV_OP_INTERNAL(ComplexFMulINTEL, internal::ComplexFMulINTEL) _SPIRV_OP_INTERNAL(ComplexFDivINTEL, internal::ComplexFDivINTEL) _SPIRV_OP_INTERNAL(MaskedGatherINTEL, internal::OpMaskedGatherINTEL) _SPIRV_OP_INTERNAL(MaskedScatterINTEL, internal::OpMaskedScatterINTEL) _SPIRV_OP_INTERNAL(RoundFToTF32INTEL, internal::RoundFToTF32INTEL) _SPIRV_OP_INTERNAL(ConvertHandleToImageINTEL, internal::ConvertHandleToImageINTEL) _SPIRV_OP_INTERNAL(ConvertHandleToSamplerINTEL, internal::ConvertHandleToSamplerINTEL) _SPIRV_OP_INTERNAL(ConvertHandleToSampledImageINTEL, internal::ConvertHandleToSampledImageINTEL) SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVStream.cpp000066400000000000000000000300161506646577400233410ustar00rootroot00000000000000//===- SPIRVStream.cpp - Class to represent a SPIR-V Stream -----*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements SPIR-V stream class. /// //===----------------------------------------------------------------------===// #include "SPIRVStream.h" #include "SPIRVDebug.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVNameMapEnum.h" #include "SPIRVOpCode.h" #include // std::numeric_limits namespace SPIRV { /// Write string with quote. Replace " with \". static void writeQuotedString(spv_ostream &O, const std::string &Str) { O << '"'; for (auto I : Str) { if (I == '"') O << '\\'; O << I; } O << '"'; } /// Read quoted string. Replace \" with ". static void readQuotedString(std::istream &IS, std::string &Str) { char Ch = ' '; char PreCh = ' '; while (IS >> Ch && Ch != '"') ; if (IS >> PreCh && PreCh != '"') { while (IS >> Ch) { if (Ch == '"') { if (PreCh != '\\') { Str += PreCh; break; } else PreCh = Ch; } else { Str += PreCh; PreCh = Ch; } } } } #ifdef _SPIRV_SUPPORT_TEXT_FMT bool SPIRVUseTextFormat = false; #endif SPIRVDecoder::SPIRVDecoder(std::istream &InputStream, SPIRVFunction &F) : IS(InputStream), M(*F.getModule()), WordCount(0), OpCode(OpNop), Scope(&F) {} SPIRVDecoder::SPIRVDecoder(std::istream &InputStream, SPIRVBasicBlock &BB) : IS(InputStream), M(*BB.getModule()), WordCount(0), OpCode(OpNop), Scope(&BB) {} void SPIRVDecoder::setScope(SPIRVEntry *TheScope) { assert(TheScope && (TheScope->getOpCode() == OpFunction || TheScope->getOpCode() == OpLabel)); Scope = TheScope; } template const SPIRVDecoder &decode(const SPIRVDecoder &I, T &V) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { std::string W; I.IS >> W; V = getNameMap(V).rmap(W); SPIRVDBG(spvdbgs() << "Read word: W = " << W << " V = " << V << '\n'); return I; } #endif return decodeBinary(I, V); } template const SPIRVEncoder &encode(const SPIRVEncoder &O, T V) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { O.OS << getNameMap(V).map(V) << " "; return O; } #endif return O << static_cast(V); } template <> const SPIRVEncoder &operator<<(const SPIRVEncoder &O, SPIRVType *P) { if (!P->hasId() && P->getOpCode() == OpTypeForwardPointer) return O << static_cast( static_cast(P)) ->getPointerId(); return O << P->getId(); } #define SPIRV_DEF_ENCDEC(Type) \ const SPIRVDecoder &operator>>(const SPIRVDecoder &I, Type &V) { \ return decode(I, V); \ } \ const SPIRVEncoder &operator<<(const SPIRVEncoder &O, Type V) { \ return encode(O, V); \ } SPIRV_DEF_ENCDEC(Op) SPIRV_DEF_ENCDEC(Capability) SPIRV_DEF_ENCDEC(Decoration) SPIRV_DEF_ENCDEC(OCLExtOpKind) SPIRV_DEF_ENCDEC(SPIRVDebugExtOpKind) SPIRV_DEF_ENCDEC(NonSemanticAuxDataOpKind) SPIRV_DEF_ENCDEC(InitializationModeQualifier) SPIRV_DEF_ENCDEC(HostAccessQualifier) SPIRV_DEF_ENCDEC(NamedMaximumNumberOfRegisters) SPIRV_DEF_ENCDEC(LinkageType) // Read a string with padded 0's at the end so that they form a stream of // words. const SPIRVDecoder &operator>>(const SPIRVDecoder &I, std::string &Str) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { readQuotedString(I.IS, Str); SPIRVDBG(spvdbgs() << "Read string: \"" << Str << "\"\n"); return I; } #endif uint64_t Count = 0; char Ch; while (I.IS.get(Ch) && Ch != '\0') { Str += Ch; ++Count; } Count = (Count + 1) % 4; Count = Count ? 4 - Count : 0; for (; Count; --Count) { I.IS >> Ch; assert(Ch == '\0' && "Invalid string in SPIRV"); } SPIRVDBG(spvdbgs() << "Read string: \"" << Str << "\"\n"); return I; } // Write a string with padded 0's at the end so that they form a stream of // words. const SPIRVEncoder &operator<<(const SPIRVEncoder &O, const std::string &Str) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { writeQuotedString(O.OS, Str); O.OS << " "; return O; } #endif size_t L = Str.length(); O.OS.write(Str.c_str(), L); char Zeros[4] = {0, 0, 0, 0}; O.OS.write(Zeros, 4 - L % 4); return O; } bool SPIRVDecoder::getWordCountAndOpCode() { if (IS.eof()) { WordCount = 0; OpCode = OpNop; SPIRVDBG(spvdbgs() << "[SPIRVDecoder] getWordCountAndOpCode EOF " << WordCount << " " << OpCode << '\n'); return false; } #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { *this >> WordCount; assert(!IS.bad() && "SPIRV stream is bad"); if (IS.fail()) { WordCount = 0; OpCode = OpNop; SPIRVDBG(spvdbgs() << "[SPIRVDecoder] getWordCountAndOpCode FAIL " << WordCount << " " << OpCode << '\n'); return false; } *this >> OpCode; } else { #endif SPIRVWord WordCountAndOpCode; *this >> WordCountAndOpCode; WordCount = WordCountAndOpCode >> 16; OpCode = static_cast(WordCountAndOpCode & 0xFFFF); #ifdef _SPIRV_SUPPORT_TEXT_FMT } #endif assert(!IS.bad() && "SPIRV stream is bad"); if (IS.fail()) { WordCount = 0; OpCode = OpNop; SPIRVDBG(spvdbgs() << "[SPIRVDecoder] getWordCountAndOpCode FAIL " << WordCount << " " << OpCode << '\n'); return false; } SPIRVDBG(spvdbgs() << "[SPIRVDecoder] getWordCountAndOpCode " << WordCount << " " << OpCodeNameMap::map(OpCode) << '\n'); return true; } SPIRVEntry *SPIRVDecoder::getEntry() { if (WordCount == 0 || OpCode == OpNop) return nullptr; SPIRVEntry *Entry = SPIRVEntry::create(OpCode); assert(Entry); Entry->setModule(&M); if (isModuleScopeAllowedOpCode(OpCode) && !Scope) { } else Entry->setScope(Scope); Entry->setWordCount(WordCount); if (OpCode != OpLine) Entry->setLine(M.getCurrentLine()); if (!Entry->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::DebugLine) && !Entry->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugLine)) Entry->setDebugLine(M.getCurrentDebugLine()); IS >> *Entry; if (Entry->isEndOfBlock() || OpCode == OpNoLine) M.setCurrentLine(nullptr); if (Entry->isEndOfBlock() || Entry->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_100, SPIRVDebug::DebugNoLine) || Entry->isExtInst(SPIRVEIS_NonSemantic_Shader_DebugInfo_200, SPIRVDebug::DebugNoLine)) M.setCurrentDebugLine(nullptr); if (OpExtension == OpCode) { auto *OpExt = static_cast(Entry); ExtensionID ExtID = {}; bool ExtIsKnown = SPIRVMap::rfind( OpExt->getExtensionName(), &ExtID); if (!M.getErrorLog().checkError( ExtIsKnown, SPIRVEC_InvalidModule, "input SPIR-V module uses unknown extension '" + OpExt->getExtensionName() + "'")) { M.setInvalid(); } if (!M.getErrorLog().checkError( M.isAllowedToUseExtension(ExtID), SPIRVEC_InvalidModule, "input SPIR-V module uses extension '" + OpExt->getExtensionName() + "' which were disabled by --spirv-ext option")) { M.setInvalid(); } } if (!M.getErrorLog().checkError(Entry->isImplemented(), SPIRVEC_UnimplementedOpCode, std::to_string(Entry->getOpCode()))) { M.setInvalid(); } assert(!IS.bad() && !IS.fail() && "SPIRV stream fails"); return Entry; } void SPIRVDecoder::validate() const { assert(OpCode != OpNop && "Invalid op code"); assert(WordCount && "Invalid word count"); assert(!IS.bad() && "Bad iInput stream"); } // Skip \param n words in SPIR-V binary stream. // In case of SPIR-V text format always skip until the end of the line. void SPIRVDecoder::ignore(size_t N) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { IS.ignore(std::numeric_limits::max(), '\n'); return; } #endif IS.ignore(N * sizeof(SPIRVWord)); } void SPIRVDecoder::ignoreInstruction() { ignore(WordCount - 1); } spv_ostream &operator<<(spv_ostream &O, const SPIRVNL &E) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) O << '\n'; #endif return O; } // Read the next word from the stream and if OpCode matches the argument, // decode the whole instruction. Multiple such instructions are possible. If // OpCode doesn't match the argument, set position of the next character to be // extracted from the stream to the beginning of the non-matching instruction. // Returns vector of extracted instructions. // Used to decode SPIRVTypeStructContinuedINTEL, // SPIRVConstantCompositeContinuedINTEL and // SPIRVSpecConstantCompositeContinuedINTEL. std::vector SPIRVDecoder::getContinuedInstructions(const spv::Op ContinuedOpCode) { std::vector ContinuedInst; std::streampos Pos = IS.tellg(); // remember position getWordCountAndOpCode(); while (OpCode == ContinuedOpCode) { SPIRVEntry *Entry = getEntry(); assert(Entry && "Failed to decode entry! Invalid instruction!"); M.add(Entry); ContinuedInst.push_back(Entry); Pos = IS.tellg(); getWordCountAndOpCode(); } IS.seekg(Pos); // restore position return ContinuedInst; } std::vector SPIRVDecoder::getSourceContinuedInstructions() { std::vector ContinuedInst; std::streampos Pos = IS.tellg(); // remember position getWordCountAndOpCode(); while (OpCode == OpExtInst) { SPIRVEntry *Entry = getEntry(); assert(Entry && "Failed to decode entry! Invalid instruction!"); SPIRVExtInst *Inst = static_cast(Entry); if (Inst->getExtOp() != SPIRVDebug::Instruction::SourceContinued) { IS.seekg(Pos); // restore position return ContinuedInst; } M.add(Entry); ContinuedInst.push_back(Entry); Pos = IS.tellg(); getWordCountAndOpCode(); } IS.seekg(Pos); // restore position return ContinuedInst; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVStream.h000066400000000000000000000164341506646577400230160ustar00rootroot00000000000000//===- SPIRVStream.h - Class to represent a SPIR-V Stream -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines Word class for SPIR-V. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVSTREAM_H #define SPIRV_LIBSPIRV_SPIRVSTREAM_H #include "SPIRVDebug.h" #include "SPIRVExtInst.h" #include "SPIRVModule.h" #include #include #include #include #include namespace SPIRV { #ifndef _SPIRV_SUPPORT_TEXT_FMT #define _SPIRV_SUPPORT_TEXT_FMT #endif #ifdef _SPIRV_SUPPORT_TEXT_FMT // Use textual format for SPIRV. extern bool SPIRVUseTextFormat; #endif class SPIRVFunction; class SPIRVBasicBlock; class SPIRVDecoder { public: SPIRVDecoder(std::istream &InputStream, SPIRVModule &Module) : IS(InputStream), M(Module), WordCount(0), OpCode(OpNop), Scope(NULL) {} SPIRVDecoder(std::istream &InputStream, SPIRVFunction &F); SPIRVDecoder(std::istream &InputStream, SPIRVBasicBlock &BB); void setScope(SPIRVEntry *); bool getWordCountAndOpCode(); SPIRVEntry *getEntry(); void validate() const; void ignore(size_t N); void ignoreInstruction(); std::vector getContinuedInstructions(const spv::Op ContinuedOpCode); std::vector getSourceContinuedInstructions(); std::istream &IS; SPIRVModule &M; SPIRVWord WordCount; Op OpCode; SPIRVEntry *Scope; // A function or basic block }; class SPIRVEncoder { public: explicit SPIRVEncoder(spv_ostream &OutputStream) : OS(OutputStream) {} spv_ostream &OS; }; /// Output a new line in text mode. Do nothing in binary mode. class SPIRVNL { friend spv_ostream &operator<<(spv_ostream &O, const SPIRVNL &E); }; template const SPIRVDecoder &decodeBinary(const SPIRVDecoder &I, T &V) { uint32_t W; I.IS.read(reinterpret_cast(&W), sizeof(W)); V = static_cast(W); SPIRVDBG(spvdbgs() << "Read word: W = " << W << " V = " << V << '\n'); return I; } #ifdef _SPIRV_SUPPORT_TEXT_FMT /// Skip comment and whitespace. Comment starts with ';', ends with '\n'. inline std::istream &skipcomment(std::istream &IS) { if (IS.eof() || IS.bad()) return IS; char C = IS.peek(); while (std::char_traits::not_eof(C) && std::isspace(C)) { IS.get(); C = IS.peek(); } while (std::char_traits::not_eof(C) && C == ';') { IS.ignore(std::numeric_limits::max(), '\n'); C = IS.peek(); while (std::char_traits::not_eof(C) && std::isspace(C)) { IS.get(); C = IS.peek(); } } return IS; } #endif template const SPIRVDecoder &operator>>(const SPIRVDecoder &I, T &V) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { uint32_t W; I.IS >> skipcomment >> W; V = static_cast(W); SPIRVDBG(spvdbgs() << "Read word: W = " << W << " V = " << V << '\n'); return I; } #endif return decodeBinary(I, V); } template const SPIRVDecoder &operator>>(const SPIRVDecoder &I, T *&P) { SPIRVId Id; I >> Id; P = static_cast(I.M.getEntry(Id)); return I; } template const SPIRVDecoder &operator>>(const SPIRVDecoder &Decoder, const std::pair &Range) { for (IterTy I = Range.first, E = Range.second; I != E; ++I) Decoder >> *I; return Decoder; } template const SPIRVDecoder &operator>>(const SPIRVDecoder &I, std::vector &V) { for (size_t J = 0, E = V.size(); J != E; ++J) I >> V[J]; return I; } template const SPIRVDecoder &operator>>(const SPIRVDecoder &I, llvm::Optional &V) { if (V) I >> V.getValue(); return I; } template const SPIRVEncoder &operator<<(const SPIRVEncoder &O, T V) { #ifdef _SPIRV_SUPPORT_TEXT_FMT if (SPIRVUseTextFormat) { O.OS << V << " "; return O; } #endif uint32_t W = static_cast(V); O.OS.write(reinterpret_cast(&W), sizeof(W)); return O; } template const SPIRVEncoder &operator<<(const SPIRVEncoder &O, T *P) { return O << P->getId(); } template <> const SPIRVEncoder &operator<<(const SPIRVEncoder &O, SPIRVType *P); template const SPIRVEncoder &operator<<(const SPIRVEncoder &O, const std::vector &V) { for (size_t I = 0, E = V.size(); I != E; ++I) O << V[I]; return O; } template const SPIRVEncoder &operator<<(const SPIRVEncoder &O, const llvm::Optional &V) { if (V) O << V.getValue(); return O; } template const SPIRVEncoder &operator<<(const SPIRVEncoder &Encoder, const std::pair &Range) { for (IterTy I = Range.first, E = Range.second; I != E; ++I) Encoder << *I; return Encoder; } #define SPIRV_DEC_ENCDEC(Type) \ const SPIRVEncoder &operator<<(const SPIRVEncoder &O, Type V); \ const SPIRVDecoder &operator>>(const SPIRVDecoder &I, Type &V); SPIRV_DEC_ENCDEC(Op) SPIRV_DEC_ENCDEC(Capability) SPIRV_DEC_ENCDEC(Decoration) SPIRV_DEC_ENCDEC(OCLExtOpKind) SPIRV_DEC_ENCDEC(SPIRVDebugExtOpKind) SPIRV_DEC_ENCDEC(NonSemanticAuxDataOpKind) SPIRV_DEC_ENCDEC(InitializationModeQualifier) SPIRV_DEC_ENCDEC(HostAccessQualifier) SPIRV_DEC_ENCDEC(NamedMaximumNumberOfRegisters) SPIRV_DEC_ENCDEC(LinkageType) const SPIRVEncoder &operator<<(const SPIRVEncoder &O, const std::string &Str); const SPIRVDecoder &operator>>(const SPIRVDecoder &I, std::string &Str); } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVSTREAM_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVType.cpp000066400000000000000000000262571506646577400230430ustar00rootroot00000000000000//===- SPIRVtype.cpp - Class to represent a SPIR-V type ---------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file implements the types defined in SPIRV spec with op codes. /// //===----------------------------------------------------------------------===// #include "SPIRVType.h" #include "SPIRVDecorate.h" #include "SPIRVModule.h" #include "SPIRVValue.h" #include namespace SPIRV { SPIRVType *SPIRVType::getArrayElementType() const { assert(OpCode == OpTypeArray && "Not array type"); return static_cast(this)->getElementType(); } uint64_t SPIRVType::getArrayLength() const { assert(OpCode == OpTypeArray && "Not array type"); const SPIRVTypeArray *AsArray = static_cast(this); assert(AsArray->getLength()->getOpCode() == OpConstant && "getArrayLength can only be called with constant array lengths"); return AsArray->getLength()->getZExtIntValue(); } SPIRVWord SPIRVType::getBitWidth() const { if (isTypeVector()) return getVectorComponentType()->getBitWidth(); if (isTypeBool()) return 1; return isTypeInt() ? getIntegerBitWidth() : getFloatBitWidth(); } SPIRVWord SPIRVType::getFloatBitWidth() const { assert(OpCode == OpTypeFloat && "Not a float type"); return static_cast(this)->getBitWidth(); } SPIRVWord SPIRVType::getIntegerBitWidth() const { assert((OpCode == OpTypeInt || OpCode == OpTypeBool) && "Not an integer type"); if (isTypeBool()) return 1; return static_cast(this)->getBitWidth(); } SPIRVType *SPIRVType::getFunctionReturnType() const { assert(OpCode == OpTypeFunction); return static_cast(this)->getReturnType(); } SPIRVType *SPIRVType::getPointerElementType() const { assert(OpCode == OpTypePointer && "Not a pointer type"); return static_cast(this)->getElementType(); } SPIRVStorageClassKind SPIRVType::getPointerStorageClass() const { assert(OpCode == OpTypePointer && "Not a pointer type"); return static_cast(this)->getStorageClass(); } SPIRVType *SPIRVType::getStructMemberType(size_t Index) const { assert(OpCode == OpTypeStruct && "Not struct type"); return static_cast(this)->getMemberType(Index); } SPIRVWord SPIRVType::getStructMemberCount() const { assert(OpCode == OpTypeStruct && "Not struct type"); return static_cast(this)->getMemberCount(); } SPIRVWord SPIRVType::getVectorComponentCount() const { assert(OpCode == OpTypeVector && "Not vector type"); return static_cast(this)->getComponentCount(); } SPIRVType *SPIRVType::getVectorComponentType() const { if (OpCode == OpTypeVector) return static_cast(this)->getComponentType(); if (OpCode == internal::OpTypeJointMatrixINTEL) return static_cast(this)->getCompType(); if (OpCode == OpTypeCooperativeMatrixKHR) return static_cast(this) ->getCompType(); assert(0 && "getVectorComponentType(): Not a vector or joint matrix type"); return nullptr; } SPIRVWord SPIRVType::getMatrixColumnCount() const { assert(OpCode == OpTypeMatrix && "Not matrix type"); return static_cast(this)->getColumnCount(); } SPIRVType *SPIRVType::getMatrixColumnType() const { assert(OpCode == OpTypeMatrix && "Not matrix type"); return static_cast(this)->getColumnType(); } SPIRVType *SPIRVType::getScalarType() const { switch (OpCode) { case OpTypePointer: return getPointerElementType()->getScalarType(); case OpTypeArray: return getArrayElementType(); case OpTypeVector: return getVectorComponentType(); case OpTypeMatrix: return getMatrixColumnType()->getVectorComponentType(); case OpTypeInt: case OpTypeFloat: case OpTypeBool: return const_cast(this); default: break; } return nullptr; } bool SPIRVType::isTypeVoid() const { return OpCode == OpTypeVoid; } bool SPIRVType::isTypeArray() const { return OpCode == OpTypeArray; } bool SPIRVType::isTypeBool() const { return OpCode == OpTypeBool; } bool SPIRVType::isTypeComposite() const { return isTypeVector() || isTypeArray() || isTypeStruct() || isTypeJointMatrixINTEL() || isTypeCooperativeMatrixKHR(); } bool SPIRVType::isTypeFloat(unsigned Bits, unsigned FloatingPointEncoding) const { if (!isType(this)) return false; if (Bits == 0) return true; const auto *ThisFloat = static_cast(this); return ThisFloat->getBitWidth() == Bits && ThisFloat->getFloatingPointEncoding() == FloatingPointEncoding; } bool SPIRVType::isTypeOCLImage() const { return isTypeImage() && static_cast(this)->isOCLImage(); } bool SPIRVType::isTypePipe() const { return OpCode == OpTypePipe; } bool SPIRVType::isTypePipeStorage() const { return OpCode == OpTypePipeStorage; } bool SPIRVType::isTypeReserveId() const { return OpCode == OpTypeReserveId; } bool SPIRVType::isTypeInt(unsigned Bits) const { return isType(this, Bits); } bool SPIRVType::isTypePointer() const { return OpCode == OpTypePointer; } bool SPIRVType::isTypeOpaque() const { return OpCode == OpTypeOpaque; } bool SPIRVType::isTypeEvent() const { return OpCode == OpTypeEvent; } bool SPIRVType::isTypeDeviceEvent() const { return OpCode == OpTypeDeviceEvent; } bool SPIRVType::isTypeSampler() const { return OpCode == OpTypeSampler; } bool SPIRVType::isTypeImage() const { return OpCode == OpTypeImage; } bool SPIRVType::isTypeSampledImage() const { return OpCode == OpTypeSampledImage; } bool SPIRVType::isTypeStruct() const { return OpCode == OpTypeStruct; } bool SPIRVType::isTypeVector() const { return OpCode == OpTypeVector; } bool SPIRVType::isTypeJointMatrixINTEL() const { return OpCode == internal::OpTypeJointMatrixINTEL; } bool SPIRVType::isTypeCooperativeMatrixKHR() const { return OpCode == OpTypeCooperativeMatrixKHR; } bool SPIRVType::isTypeVectorBool() const { return isTypeVector() && getVectorComponentType()->isTypeBool(); } bool SPIRVType::isTypeVectorInt() const { return isTypeVector() && getVectorComponentType()->isTypeInt(); } bool SPIRVType::isTypeVectorFloat() const { return isTypeVector() && getVectorComponentType()->isTypeFloat(); } bool SPIRVType::isTypeVectorOrScalarBool() const { return isTypeBool() || isTypeVectorBool(); } bool SPIRVType::isTypeVectorPointer() const { return isTypeVector() && getVectorComponentType()->isTypePointer(); } bool SPIRVType::isTypeSubgroupAvcINTEL() const { return isSubgroupAvcINTELTypeOpCode(OpCode); } bool SPIRVType::isTypeSubgroupAvcMceINTEL() const { return OpCode == OpTypeAvcMcePayloadINTEL || OpCode == OpTypeAvcMceResultINTEL; } bool SPIRVType::isTypeVectorOrScalarInt() const { return isTypeInt() || isTypeVectorInt(); } bool SPIRVType::isTypeVectorOrScalarFloat() const { return isTypeFloat() || isTypeVectorFloat(); } bool SPIRVTypeStruct::isPacked() const { return hasDecorate(DecorationCPacked); } void SPIRVTypeStruct::setPacked(bool Packed) { if (Packed) addDecorate(new SPIRVDecorate(DecorationCPacked, this)); else eraseDecorate(DecorationCPacked); } SPIRVTypeArray::SPIRVTypeArray(SPIRVModule *M, SPIRVId TheId, SPIRVType *TheElemType, SPIRVConstant *TheLength) : SPIRVType(M, 4, OpTypeArray, TheId), ElemType(TheElemType), Length(TheLength->getId()) { validate(); } void SPIRVTypeArray::validate() const { SPIRVEntry::validate(); ElemType->validate(); assert(getValue(Length)->getType()->isTypeInt()); } SPIRVConstant *SPIRVTypeArray::getLength() const { return get(Length); } _SPIRV_IMP_ENCDEC3(SPIRVTypeArray, Id, ElemType, Length) void SPIRVTypeForwardPointer::encode(spv_ostream &O) const { getEncoder(O) << PointerId << SC; } void SPIRVTypeForwardPointer::decode(std::istream &I) { auto Decoder = getDecoder(I); Decoder >> PointerId >> SC; } SPIRVTypeJointMatrixINTEL::SPIRVTypeJointMatrixINTEL( SPIRVModule *M, SPIRVId TheId, SPIRVType *CompType, std::vector Args) : SPIRVType(M, FixedWC + Args.size(), OC, TheId), CompType(CompType), Args(Args) {} SPIRVTypeJointMatrixINTEL::SPIRVTypeJointMatrixINTEL() : SPIRVType(OC), CompType(nullptr), Args({nullptr, nullptr, nullptr, nullptr}) {} void SPIRVTypeJointMatrixINTEL::encode(spv_ostream &O) const { auto Encoder = getEncoder(O); Encoder << Id << CompType << Args; } void SPIRVTypeJointMatrixINTEL::decode(std::istream &I) { auto Decoder = getDecoder(I); Decoder >> Id >> CompType >> Args; } SPIRVTypeCooperativeMatrixKHR::SPIRVTypeCooperativeMatrixKHR( SPIRVModule *M, SPIRVId TheId, SPIRVType *CompType, std::vector Args) : SPIRVType(M, FixedWC, OpTypeCooperativeMatrixKHR, TheId), CompType(CompType), Args(std::move(Args)) {} SPIRVTypeCooperativeMatrixKHR::SPIRVTypeCooperativeMatrixKHR() : SPIRVType(OpTypeCooperativeMatrixKHR), CompType(nullptr), Args({nullptr, nullptr, nullptr, nullptr}) {} void SPIRVTypeCooperativeMatrixKHR::encode(spv_ostream &O) const { auto Encoder = getEncoder(O); Encoder << Id << CompType << Args; } void SPIRVTypeCooperativeMatrixKHR::decode(std::istream &I) { auto Decoder = getDecoder(I); Decoder >> Id >> CompType >> Args; } } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVType.h000066400000000000000000001140551506646577400225020ustar00rootroot00000000000000//===- SPIRVType.h - Class to represent a SPIR-V Type -----------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the types defined in SPIRV spec with op codes. /// /// The name of the SPIR-V types follow the op code name in the spec, e.g. /// SPIR-V type with op code name OpTypeInt is named as SPIRVTypeInt. This is /// for readability and ease of using macro to handle types. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVTYPE_H #define SPIRV_LIBSPIRV_SPIRVTYPE_H #include "SPIRVEntry.h" #include "SPIRVStream.h" #include #include #include namespace SPIRV { class SPIRVType : public SPIRVEntry { public: // Complete constructor SPIRVType(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode, SPIRVId TheId) : SPIRVEntry(M, TheWordCount, TheOpCode, TheId) {} // Incomplete constructor SPIRVType(Op TheOpCode) : SPIRVEntry(TheOpCode) {} SPIRVType *getArrayElementType() const; uint64_t getArrayLength() const; unsigned getBitWidth() const; unsigned getFloatBitWidth() const; SPIRVType *getFunctionReturnType() const; unsigned getIntegerBitWidth() const; SPIRVType *getPointerElementType() const; SPIRVStorageClassKind getPointerStorageClass() const; SPIRVType *getStructMemberType(size_t) const; SPIRVWord getStructMemberCount() const; SPIRVWord getVectorComponentCount() const; SPIRVType *getVectorComponentType() const; SPIRVWord getMatrixColumnCount() const; SPIRVType *getMatrixColumnType() const; SPIRVType *getScalarType() const; bool isTypeVoid() const; bool isTypeArray() const; bool isTypeBool() const; bool isTypeComposite() const; bool isTypeEvent() const; bool isTypeDeviceEvent() const; bool isTypeReserveId() const; bool isTypeFloat(unsigned Bits = 0, unsigned FloatingPointEncoding = FPEncodingMax) const; bool isTypeImage() const; bool isTypeOCLImage() const; bool isTypePipe() const; bool isTypePipeStorage() const; bool isTypeInt(unsigned Bits = 0) const; bool isTypeOpaque() const; bool isTypePointer() const; bool isTypeSampler() const; bool isTypeSampledImage() const; bool isTypeStruct() const; bool isTypeVector() const; bool isTypeJointMatrixINTEL() const; bool isTypeCooperativeMatrixKHR() const; bool isTypeVectorInt() const; bool isTypeVectorFloat() const; bool isTypeVectorBool() const; bool isTypeVectorOrScalarInt() const; bool isTypeVectorOrScalarFloat() const; bool isTypeVectorOrScalarBool() const; bool isTypeVectorPointer() const; bool isTypeSubgroupAvcINTEL() const; bool isTypeSubgroupAvcMceINTEL() const; }; class SPIRVTypeVoid : public SPIRVType { public: // Complete constructor SPIRVTypeVoid(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, OpTypeVoid, TheId) {} // Incomplete constructor SPIRVTypeVoid() : SPIRVType(OpTypeVoid) {} protected: _SPIRV_DEF_ENCDEC1(Id) }; class SPIRVTypeBool : public SPIRVType { public: // Complete constructor SPIRVTypeBool(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, OpTypeBool, TheId) {} // Incomplete constructor SPIRVTypeBool() : SPIRVType(OpTypeBool) {} protected: _SPIRV_DEF_ENCDEC1(Id) }; class SPIRVTypeInt : public SPIRVType { public: static const Op OC = OpTypeInt; // Complete constructor SPIRVTypeInt(SPIRVModule *M, SPIRVId TheId, unsigned TheBitWidth, bool ItIsSigned) : SPIRVType(M, 4, OC, TheId), BitWidth(TheBitWidth), IsSigned(ItIsSigned) { validate(); } // Incomplete constructor SPIRVTypeInt() : SPIRVType(OC), BitWidth(0), IsSigned(false) {} unsigned getBitWidth() const { return BitWidth; } bool isSigned() const { return IsSigned; } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec CV; switch (BitWidth) { case 4: { if (Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_int4)) { CV.push_back(CapabilityInt4TypeINTEL); return CV; } CV.push_back(CapabilityArbitraryPrecisionIntegersINTEL); return CV; } case 8: CV.push_back(CapabilityInt8); break; case 16: CV.push_back(CapabilityInt16); break; case 32: break; case 64: CV.push_back(CapabilityInt64); break; default: if (Module->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_integers)) CV.push_back(CapabilityArbitraryPrecisionIntegersINTEL); } return CV; } llvm::Optional getRequiredExtension() const override { switch (BitWidth) { case 4: { if (Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_int4)) return ExtensionID::SPV_INTEL_int4; return ExtensionID::SPV_INTEL_arbitrary_precision_integers; } case 8: case 16: case 32: case 64: return {}; default: return ExtensionID::SPV_INTEL_arbitrary_precision_integers; } } protected: _SPIRV_DEF_ENCDEC3(Id, BitWidth, IsSigned) void validate() const override { SPIRVEntry::validate(); assert(((BitWidth == 4 && Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_int4)) || BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64 || Module->isAllowedToUseExtension( ExtensionID::SPV_INTEL_arbitrary_precision_integers)) && "Invalid bit width"); } private: unsigned BitWidth; // Bit width bool IsSigned; // Whether it is signed }; class SPIRVTypeFloat : public SPIRVType { public: static const Op OC = OpTypeFloat; // Complete constructor SPIRVTypeFloat(SPIRVModule *M, SPIRVId TheId, unsigned TheBitWidth, unsigned TheFloatingPointEncoding) : SPIRVType(M, 3 + (TheFloatingPointEncoding != FPEncodingMax), OC, TheId), BitWidth(TheBitWidth), FloatingPointEncoding(TheFloatingPointEncoding) { } // Incomplete constructor SPIRVTypeFloat() : SPIRVType(OC), BitWidth(0), FloatingPointEncoding(FPEncodingMax) {} unsigned getBitWidth() const { return BitWidth; } unsigned getFloatingPointEncoding() const { return FloatingPointEncoding; } llvm::Optional getRequiredExtension() const override { if (isTypeFloat(16, FPEncodingBFloat16KHR)) return ExtensionID::SPV_KHR_bfloat16; return {}; } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec CV; if (isTypeFloat(16, FPEncodingBFloat16KHR)) { CV.push_back(CapabilityBFloat16TypeKHR); } else if (isTypeFloat(16)) { CV.push_back(CapabilityFloat16Buffer); auto Extensions = getModule()->getSourceExtension(); if (std::any_of(Extensions.begin(), Extensions.end(), [](const std::string &I) { return I == "cl_khr_fp16"; })) CV.push_back(CapabilityFloat16); } else if (isTypeFloat(64)) CV.push_back(CapabilityFloat64); return CV; } protected: void encode(spv_ostream &O) const override { assert(WordCount == 3 || WordCount == 4); auto Encoder = getEncoder(O); Encoder << Id << BitWidth; if (WordCount == 4) Encoder << FloatingPointEncoding; } void decode(std::istream &I) override { assert(WordCount == 3 || WordCount == 4); auto Decoder = getDecoder(I); Decoder >> Id >> BitWidth; if (WordCount == 4) Decoder >> FloatingPointEncoding; } void validate() const override { SPIRVEntry::validate(); assert(BitWidth >= 16 && BitWidth <= 64 && "Invalid bit width"); assert( (FloatingPointEncoding == FPEncodingMax || (BitWidth == 16 && FloatingPointEncoding == FPEncodingBFloat16KHR)) && "Invalid floating point encoding"); } private: unsigned BitWidth; // Bit width unsigned FloatingPointEncoding; }; class SPIRVTypePointer : public SPIRVType { public: // Complete constructor SPIRVTypePointer(SPIRVModule *M, SPIRVId TheId, SPIRVStorageClassKind TheStorageClass, SPIRVType *ElementType) : SPIRVType(M, 4, OpTypePointer, TheId), ElemStorageClass(TheStorageClass), ElemTypeId(ElementType->getId()) { validate(); } // Incomplete constructor SPIRVTypePointer() : SPIRVType(OpTypePointer), ElemStorageClass(StorageClassFunction), ElemTypeId(0) {} SPIRVType *getElementType() const { return static_cast(getEntry(ElemTypeId)); } SPIRVStorageClassKind getStorageClass() const { return ElemStorageClass; } SPIRVCapVec getRequiredCapability() const override { auto Cap = getVec(CapabilityAddresses); if (getElementType()->isTypeFloat(16)) Cap.push_back(CapabilityFloat16Buffer); auto C = getCapability(ElemStorageClass); Cap.insert(Cap.end(), C.begin(), C.end()); return Cap; } std::vector getNonLiteralOperands() const override { return std::vector(1, getEntry(ElemTypeId)); } protected: _SPIRV_DEF_ENCDEC3(Id, ElemStorageClass, ElemTypeId) void validate() const override { SPIRVEntry::validate(); assert(isValid(ElemStorageClass)); } private: SPIRVStorageClassKind ElemStorageClass; // Storage Class SPIRVId ElemTypeId; }; class SPIRVTypeForwardPointer : public SPIRVEntryNoId { public: SPIRVTypeForwardPointer(SPIRVModule *M, SPIRVId PointerId, SPIRVStorageClassKind SC) : SPIRVEntryNoId(M, 3), PointerId(PointerId), SC(SC) {} SPIRVTypeForwardPointer() : PointerId(SPIRVID_INVALID), SC(StorageClassUniformConstant) {} SPIRVId getPointerId() const { return PointerId; } _SPIRV_DCL_ENCDEC private: SPIRVId PointerId; SPIRVStorageClassKind SC; }; class SPIRVTypeVector : public SPIRVType { public: // Complete constructor SPIRVTypeVector(SPIRVModule *M, SPIRVId TheId, SPIRVType *TheCompType, SPIRVWord TheCompCount) : SPIRVType(M, 4, OpTypeVector, TheId), CompType(TheCompType), CompCount(TheCompCount) { validate(); } // Incomplete constructor SPIRVTypeVector() : SPIRVType(OpTypeVector), CompType(nullptr), CompCount(0) {} SPIRVType *getComponentType() const { return CompType; } SPIRVWord getComponentCount() const { return CompCount; } bool isValidIndex(SPIRVWord Index) const { return Index < CompCount; } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec V(getComponentType()->getRequiredCapability()); // Even though the capability name is "Vector16", it describes // usage of 8-component or 16-component vectors. if (CompCount == 8 || CompCount == 16) V.push_back(CapabilityVector16); if (Module->isAllowedToUseExtension(ExtensionID::SPV_INTEL_vector_compute)) if (CompCount == 1 || (CompCount > 4 && CompCount < 8) || (CompCount > 8 && CompCount < 16) || CompCount > 16) V.push_back(CapabilityVectorAnyINTEL); return V; } std::vector getNonLiteralOperands() const override { return std::vector(1, CompType); } protected: _SPIRV_DEF_ENCDEC3(Id, CompType, CompCount) void validate() const override { SPIRVEntry::validate(); CompType->validate(); #ifndef NDEBUG if (!(Module->isAllowedToUseExtension( ExtensionID::SPV_INTEL_vector_compute))) { assert(CompCount == 2 || CompCount == 3 || CompCount == 4 || CompCount == 8 || CompCount == 16); } #endif // !NDEBUG } private: SPIRVType *CompType; // Component Type SPIRVWord CompCount; // Component Count }; class SPIRVTypeMatrix : public SPIRVType { public: // Complete constructor SPIRVTypeMatrix(SPIRVModule *M, SPIRVId TheId, SPIRVType *TheColType, SPIRVWord TheColCount) : SPIRVType(M, 4, OpTypeMatrix, TheId), ColType(TheColType), ColCount(TheColCount) { validate(); } // Incomplete constructor SPIRVTypeMatrix() : SPIRVType(OpTypeMatrix), ColType(nullptr), ColCount(0) {} SPIRVType *getColumnType() const { return ColType; } SPIRVWord getColumnCount() const { return ColCount; } bool isValidIndex(SPIRVWord Index) const { return Index < ColCount; } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec V(getColumnType()->getRequiredCapability()); if (ColCount >= 8) V.push_back(CapabilityVector16); return V; } virtual std::vector getNonLiteralOperands() const override { return std::vector(1, ColType); } void validate() const override { SPIRVEntry::validate(); ColType->validate(); assert(ColCount >= 2); } protected: _SPIRV_DEF_ENCDEC3(Id, ColType, ColCount) private: SPIRVType *ColType; // Column Type SPIRVWord ColCount; // Column Count }; class SPIRVTypeArray : public SPIRVType { public: // Complete constructor SPIRVTypeArray(SPIRVModule *M, SPIRVId TheId, SPIRVType *TheElemType, SPIRVConstant *TheLength); // Incomplete constructor SPIRVTypeArray() : SPIRVType(OpTypeArray), ElemType(nullptr), Length(SPIRVID_INVALID) {} SPIRVType *getElementType() const { return ElemType; } SPIRVConstant *getLength() const; SPIRVCapVec getRequiredCapability() const override { return getElementType()->getRequiredCapability(); } std::vector getNonLiteralOperands() const override { std::vector Operands(2, ElemType); Operands[1] = (SPIRVEntry *)getLength(); return Operands; } protected: _SPIRV_DCL_ENCDEC void validate() const override; private: SPIRVType *ElemType; // Element Type SPIRVId Length; // Array Length }; class SPIRVTypeOpaque : public SPIRVType { public: // Complete constructor SPIRVTypeOpaque(SPIRVModule *M, SPIRVId TheId, const std::string &TheName) : SPIRVType(M, 2 + getSizeInWords(TheName), OpTypeOpaque, TheId) { Name = TheName; validate(); } // Incomplete constructor SPIRVTypeOpaque() : SPIRVType(OpTypeOpaque) {} protected: _SPIRV_DEF_ENCDEC2(Id, Name) void validate() const override { SPIRVEntry::validate(); } }; struct SPIRVTypeImageDescriptor { SPIRVImageDimKind Dim; SPIRVWord Depth; SPIRVWord Arrayed; SPIRVWord MS; SPIRVWord Sampled; SPIRVWord Format; static std::tuple< std::tuple, SPIRVWord> getAsTuple(const SPIRVTypeImageDescriptor &Desc) { return std::make_tuple(std::make_tuple(Desc.Dim, Desc.Depth, Desc.Arrayed, Desc.MS, Desc.Sampled), Desc.Format); } SPIRVTypeImageDescriptor() : Dim(Dim1D), Depth(0), Arrayed(0), MS(0), Sampled(0), Format(0) {} SPIRVTypeImageDescriptor(SPIRVImageDimKind Dim, SPIRVWord Cont, SPIRVWord Arr, SPIRVWord Comp, SPIRVWord Mult, SPIRVWord F) : Dim(Dim), Depth(Cont), Arrayed(Arr), MS(Comp), Sampled(Mult), Format(F) {} }; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, ...) \ { \ SPIRVTypeImageDescriptor S(__VA_ARGS__); \ add(#x, S); \ } _SPIRV_OP(image1d_t, Dim1D, 0, 0, 0, 0, 0) _SPIRV_OP(image1d_buffer_t, DimBuffer, 0, 0, 0, 0, 0) _SPIRV_OP(image1d_array_t, Dim1D, 0, 1, 0, 0, 0) _SPIRV_OP(image2d_t, Dim2D, 0, 0, 0, 0, 0) _SPIRV_OP(image2d_array_t, Dim2D, 0, 1, 0, 0, 0) _SPIRV_OP(image2d_depth_t, Dim2D, 1, 0, 0, 0, 0) _SPIRV_OP(image2d_array_depth_t, Dim2D, 1, 1, 0, 0, 0) _SPIRV_OP(image2d_msaa_t, Dim2D, 0, 0, 1, 0, 0) _SPIRV_OP(image2d_array_msaa_t, Dim2D, 0, 1, 1, 0, 0) _SPIRV_OP(image2d_msaa_depth_t, Dim2D, 1, 0, 1, 0, 0) _SPIRV_OP(image2d_array_msaa_depth_t, Dim2D, 1, 1, 1, 0, 0) _SPIRV_OP(image3d_t, Dim3D, 0, 0, 0, 0, 0) #undef _SPIRV_OP } typedef SPIRVMap OCLSPIRVImageTypeMap; // Comparision function required to use the struct as map key. inline bool operator<(const SPIRVTypeImageDescriptor &A, const SPIRVTypeImageDescriptor &B) { return SPIRVTypeImageDescriptor::getAsTuple(A) < SPIRVTypeImageDescriptor::getAsTuple(B); } class SPIRVTypeImage : public SPIRVType { public: const static Op OC = OpTypeImage; constexpr static SPIRVWord FixedWC = 9; SPIRVTypeImage(SPIRVModule *M, SPIRVId TheId, SPIRVId TheSampledType, const SPIRVTypeImageDescriptor &TheDesc) : SPIRVType(M, FixedWC, OC, TheId), SampledType(TheSampledType), Desc(TheDesc) { validate(); } SPIRVTypeImage(SPIRVModule *M, SPIRVId TheId, SPIRVId TheSampledType, const SPIRVTypeImageDescriptor &TheDesc, SPIRVAccessQualifierKind TheAcc) : SPIRVType(M, FixedWC + 1, OC, TheId), SampledType(TheSampledType), Desc(TheDesc) { Acc.push_back(TheAcc); validate(); } SPIRVTypeImage() : SPIRVType(OC), SampledType(SPIRVID_INVALID), Desc() {} const SPIRVTypeImageDescriptor &getDescriptor() const { return Desc; } bool isOCLImage() const { return Desc.Sampled == 0 && Desc.Format == 0; } bool hasAccessQualifier() const { return !Acc.empty(); } SPIRVAccessQualifierKind getAccessQualifier() const { assert(hasAccessQualifier()); return Acc[0]; } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec CV; CV.push_back(CapabilityImageBasic); if (Desc.Dim == SPIRVImageDimKind::Dim1D) CV.push_back(CapabilitySampled1D); else if (Desc.Dim == SPIRVImageDimKind::DimBuffer) CV.push_back(CapabilitySampledBuffer); if (Acc.size() > 0 && Acc[0] == AccessQualifierReadWrite) CV.push_back(CapabilityImageReadWrite); if (Desc.MS) CV.push_back(CapabilityImageMipmap); return CV; } SPIRVType *getSampledType() const { return get(SampledType); } std::vector getNonLiteralOperands() const override { return std::vector(1, get(SampledType)); } protected: _SPIRV_DEF_ENCDEC9(Id, SampledType, Desc.Dim, Desc.Depth, Desc.Arrayed, Desc.MS, Desc.Sampled, Desc.Format, Acc) // The validation assumes OpenCL image or sampler type. void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC + Acc.size()); assert(SampledType != SPIRVID_INVALID && "Invalid sampled type"); assert(Desc.Dim <= 5); assert(Desc.Depth <= 1); assert(Desc.Arrayed <= 1); assert(Desc.MS <= 1); assert(Desc.Sampled == 0); // For OCL only assert(Desc.Format == 0); // For OCL only assert(Acc.size() <= 1); } void setWordCount(SPIRVWord TheWC) override { WordCount = TheWC; Acc.resize(WordCount - FixedWC); } private: SPIRVId SampledType; SPIRVTypeImageDescriptor Desc; std::vector Acc; }; class SPIRVTypeSampler : public SPIRVType { public: const static Op OC = OpTypeSampler; const static SPIRVWord FixedWC = 2; SPIRVTypeSampler(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, FixedWC, OC, TheId) { validate(); } SPIRVTypeSampler() : SPIRVType(OC) {} protected: _SPIRV_DEF_ENCDEC1(Id) void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC); } }; class SPIRVTypeSampledImage : public SPIRVType { public: const static Op OC = OpTypeSampledImage; const static SPIRVWord FixedWC = 3; SPIRVTypeSampledImage(SPIRVModule *M, SPIRVId TheId, SPIRVTypeImage *TheImgTy) : SPIRVType(M, FixedWC, OC, TheId), ImgTy(TheImgTy) { validate(); } SPIRVTypeSampledImage() : SPIRVType(OC), ImgTy(nullptr) {} const SPIRVTypeImage *getImageType() const { return ImgTy; } void setImageType(SPIRVTypeImage *TheImgTy) { ImgTy = TheImgTy; } std::vector getNonLiteralOperands() const override { return std::vector(1, ImgTy); } protected: SPIRVTypeImage *ImgTy; _SPIRV_DEF_ENCDEC2(Id, ImgTy) void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC); assert(ImgTy && ImgTy->isTypeImage()); } }; class SPIRVTypePipeStorage : public SPIRVType { public: const static Op OC = OpTypePipeStorage; const static SPIRVWord FixedWC = 2; SPIRVTypePipeStorage(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, FixedWC, OC, TheId) { validate(); } SPIRVTypePipeStorage() : SPIRVType(OC) {} protected: _SPIRV_DEF_ENCDEC1(Id) void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC); } }; class SPIRVTypeStruct : public SPIRVType { public: const static Op OC = OpTypeStruct; // There are always 2 words in this instruction except member types: // 1) WordCount + OpCode // 2) Result Id constexpr static SPIRVWord FixedWC = 2; using ContinuedInstType = typename InstToContinued::Type; // Complete constructor SPIRVTypeStruct(SPIRVModule *M, SPIRVId TheId, const std::vector &TheMemberTypes, const std::string &TheName) : SPIRVType(M, FixedWC + TheMemberTypes.size(), OC, TheId) { MemberTypeIdVec.resize(TheMemberTypes.size()); for (auto &T : TheMemberTypes) MemberTypeIdVec.push_back(T->getId()); Name = TheName; validate(); } SPIRVTypeStruct(SPIRVModule *M, SPIRVId TheId, unsigned NumMembers, const std::string &TheName) : SPIRVType(M, FixedWC + NumMembers, OC, TheId) { Name = TheName; validate(); MemberTypeIdVec.resize(NumMembers); } // Incomplete constructor SPIRVTypeStruct() : SPIRVType(OC) {} SPIRVWord getMemberCount() const { return MemberTypeIdVec.size(); } SPIRVType *getMemberType(size_t I) const { return static_cast(getEntry(MemberTypeIdVec[I])); } void setMemberType(size_t I, SPIRVType *Ty) { if (I >= MemberTypeIdVec.size() && !ContinuedInstructions.empty()) { const size_t MaxNumElements = MaxWordCount - FixedWC; I -= MaxNumElements; // Remove operands that included into OpTypeStruct ContinuedInstructions[I / MaxNumElements]->setElementId( I % MaxNumElements, Ty->getId()); } else { MemberTypeIdVec[I] = Ty->getId(); } } bool isPacked() const; void setPacked(bool Packed); void setWordCount(SPIRVWord WordCount) override { SPIRVType::setWordCount(WordCount); MemberTypeIdVec.resize(WordCount - FixedWC); } // TODO: Should we attach operands of continued instructions as well? std::vector getNonLiteralOperands() const override { std::vector Operands(MemberTypeIdVec.size()); for (size_t I = 0, E = MemberTypeIdVec.size(); I < E; ++I) Operands[I] = getEntry(MemberTypeIdVec[I]); return Operands; } void addContinuedInstruction(ContinuedInstType Inst) { ContinuedInstructions.push_back(Inst); } void encodeChildren(spv_ostream &O) const override { O << SPIRVNL(); for (auto &I : ContinuedInstructions) O << *I; } std::vector getContinuedInstructions() { return ContinuedInstructions; } protected: void encode(spv_ostream &O) const override { getEncoder(O) << Id << MemberTypeIdVec; } void decode(std::istream &I) override { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Id >> MemberTypeIdVec; Module->add(this); for (SPIRVEntry *E : Decoder.getContinuedInstructions(ContinuedOpCode)) { addContinuedInstruction(static_cast(E)); } } void validate() const override { SPIRVEntry::validate(); } private: std::vector MemberTypeIdVec; // Member Type Ids std::vector ContinuedInstructions; const spv::Op ContinuedOpCode = InstToContinued::OpCode; }; class SPIRVTypeFunction : public SPIRVType { public: // Complete constructor SPIRVTypeFunction(SPIRVModule *M, SPIRVId TheId, SPIRVType *TheReturnType, const std::vector &TheParameterTypes) : SPIRVType(M, 3 + TheParameterTypes.size(), OpTypeFunction, TheId), ReturnType(TheReturnType) { for (const SPIRVType *T : TheParameterTypes) { ParamTypeIdVec.push_back(T->getId()); } validate(); } // Incomplete constructor SPIRVTypeFunction() : SPIRVType(OpTypeFunction), ReturnType(NULL) {} SPIRVType *getReturnType() const { return ReturnType; } SPIRVWord getNumParameters() const { return ParamTypeIdVec.size(); } SPIRVType *getParameterType(unsigned I) const { return static_cast(getEntry(ParamTypeIdVec[I])); } std::vector getNonLiteralOperands() const override { std::vector Operands = {ReturnType}; for (SPIRVId I : ParamTypeIdVec) Operands.push_back(getEntry(I)); return Operands; } protected: _SPIRV_DEF_ENCDEC3(Id, ReturnType, ParamTypeIdVec) void setWordCount(SPIRVWord WordCount) override { SPIRVType::setWordCount(WordCount); ParamTypeIdVec.resize(WordCount - 3); } void validate() const override { SPIRVEntry::validate(); ReturnType->validate(); for (auto I : ParamTypeIdVec) getEntry(I)->validate(); } private: SPIRVType *ReturnType; // Return Type std::vector ParamTypeIdVec; // Parameter Type Ids }; class SPIRVTypeOpaqueGeneric : public SPIRVType { public: // Complete constructor SPIRVTypeOpaqueGeneric(Op TheOpCode, SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, TheOpCode, TheId) { validate(); } // Incomplete constructor SPIRVTypeOpaqueGeneric(Op TheOpCode) : SPIRVType(TheOpCode), Opn(SPIRVID_INVALID) {} SPIRVValue *getOperand() { return getValue(Opn); } protected: _SPIRV_DEF_ENCDEC1(Id) void validate() const override { SPIRVEntry::validate(); } SPIRVId Opn; }; template class SPIRVOpaqueGenericType : public SPIRVTypeOpaqueGeneric { public: // Complete constructor SPIRVOpaqueGenericType(SPIRVModule *M, SPIRVId TheId) : SPIRVTypeOpaqueGeneric(TheOpCode, M, TheId) {} // Incomplete constructor SPIRVOpaqueGenericType() : SPIRVTypeOpaqueGeneric(TheOpCode) {} }; #define _SPIRV_OP(x) typedef SPIRVOpaqueGenericType SPIRVType##x; _SPIRV_OP(Event) _SPIRV_OP(ReserveId) #undef _SPIRV_OP class SPIRVTypeDeviceEvent : public SPIRVType { public: // Complete constructor SPIRVTypeDeviceEvent(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, OpTypeDeviceEvent, TheId) { validate(); } // Incomplete constructor SPIRVTypeDeviceEvent() : SPIRVType(OpTypeDeviceEvent) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityDeviceEnqueue); } protected: _SPIRV_DEF_ENCDEC1(Id) void validate() const override { SPIRVEntry::validate(); } }; class SPIRVTypeQueue : public SPIRVType { public: // Complete constructor SPIRVTypeQueue(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, OpTypeQueue, TheId) { validate(); } // Incomplete constructor SPIRVTypeQueue() : SPIRVType(OpTypeQueue) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityDeviceEnqueue); } protected: _SPIRV_DEF_ENCDEC1(Id) }; class SPIRVTypePipe : public SPIRVType { public: // Complete constructor SPIRVTypePipe(SPIRVModule *M, SPIRVId TheId, SPIRVAccessQualifierKind AccessQual = AccessQualifierReadOnly) : SPIRVType(M, 3, OpTypePipe, TheId), AccessQualifier(AccessQual) { validate(); } // Incomplete constructor SPIRVTypePipe() : SPIRVType(OpTypePipe), AccessQualifier(AccessQualifierReadOnly) {} SPIRVAccessQualifierKind getAccessQualifier() const { return AccessQualifier; } void setPipeAcessQualifier(SPIRVAccessQualifierKind AccessQual) { AccessQualifier = AccessQual; assert(isValid(AccessQualifier)); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityPipes); } protected: _SPIRV_DEF_ENCDEC2(Id, AccessQualifier) void validate() const override { SPIRVEntry::validate(); } private: SPIRVAccessQualifierKind AccessQualifier; // Access Qualifier }; template bool isType(const T1 *Ty, unsigned Bits = 0) { bool Is = Ty->getOpCode() == T2::OC; if (!Is) return false; if (Bits == 0) return true; return static_cast(Ty)->getBitWidth() == Bits; } class SPIRVTypeBufferSurfaceINTEL : public SPIRVType { public: const static Op OC = OpTypeBufferSurfaceINTEL; const static SPIRVWord FixedWC = 2; SPIRVTypeBufferSurfaceINTEL(SPIRVModule *M, SPIRVId TheId, SPIRVAccessQualifierKind TheAccess) : SPIRVType(M, FixedWC + 1, OC, TheId), AccessKind(TheAccess) { validate(); } SPIRVTypeBufferSurfaceINTEL(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, FixedWC, OC, TheId) { validate(); } SPIRVTypeBufferSurfaceINTEL() : SPIRVType(OC) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityVectorComputeINTEL); } llvm::Optional getRequiredExtension() const override { return {ExtensionID::SPV_INTEL_vector_compute}; } bool hasAccessQualifier() const { return AccessKind.hasValue(); } SPIRVAccessQualifierKind getAccessQualifier() const { assert(hasAccessQualifier()); return AccessKind.getValue(); } protected: _SPIRV_DEF_ENCDEC2(Id, AccessKind) void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC + (AccessKind ? 1 : 0)); } void setWordCount(SPIRVWord TheWC) override { if (TheWC > FixedWC) AccessKind = SPIRVAccessQualifierKind::AccessQualifierMax; WordCount = TheWC; } private: llvm::Optional AccessKind; }; // SPV_INTEL_device_side_avc_motion_estimation extension types class SPIRVTypeVmeImageINTEL : public SPIRVType { public: const static Op OC = OpTypeVmeImageINTEL; const static SPIRVWord FixedWC = 3; SPIRVTypeVmeImageINTEL(SPIRVModule *M, SPIRVId TheId, SPIRVTypeImage *TheImgTy) : SPIRVType(M, FixedWC, OC, TheId), ImgTy(TheImgTy) { validate(); } SPIRVTypeVmeImageINTEL() : SPIRVType(OC), ImgTy(nullptr) {} const SPIRVTypeImage *getImageType() const { return ImgTy; } void setImageType(SPIRVTypeImage *TheImgTy) { ImgTy = TheImgTy; } virtual std::vector getNonLiteralOperands() const override { return std::vector(1, ImgTy); } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupAvcMotionEstimationINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_device_side_avc_motion_estimation; } protected: SPIRVTypeImage *ImgTy; _SPIRV_DEF_ENCDEC2(Id, ImgTy) void validate() const override { assert(OpCode == OC); assert(WordCount == FixedWC); assert(ImgTy && ImgTy->isTypeImage()); } }; class SPIRVTypeSubgroupINTEL; template <> inline void SPIRVMap::init() { #define _SPIRV_OP(x, y) \ add("opencl.intel_sub_group_avc_" #x, OpTypeAvc##y##INTEL); _SPIRV_OP(mce_payload_t, McePayload) _SPIRV_OP(mce_result_t, MceResult) _SPIRV_OP(sic_payload_t, SicPayload) _SPIRV_OP(sic_result_t, SicResult) _SPIRV_OP(ime_result_single_reference_streamout_t, ImeResultSingleReferenceStreamout) _SPIRV_OP(ime_result_dual_reference_streamout_t, ImeResultDualReferenceStreamout) _SPIRV_OP(ime_single_reference_streamin_t, ImeSingleReferenceStreamin) _SPIRV_OP(ime_dual_reference_streamin_t, ImeDualReferenceStreamin) _SPIRV_OP(ime_payload_t, ImePayload) _SPIRV_OP(ime_result_t, ImeResult) _SPIRV_OP(ref_payload_t, RefPayload) _SPIRV_OP(ref_result_t, RefResult); #undef _SPIRV_OP } typedef SPIRVMap OCLSubgroupINTELTypeOpCodeMap; class SPIRVTypeSubgroupAvcINTEL : public SPIRVType { public: // Complete constructor SPIRVTypeSubgroupAvcINTEL(Op TheOpCode, SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, TheOpCode, TheId) { validate(); } // Incomplete constructor SPIRVTypeSubgroupAvcINTEL(Op TheOpCode) : SPIRVType(TheOpCode), Opn(SPIRVID_INVALID) {} SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilitySubgroupAvcMotionEstimationINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_device_side_avc_motion_estimation; } SPIRVValue *getOperand() { return getValue(Opn); } protected: _SPIRV_DEF_ENCDEC1(Id) void validate() const override { SPIRVEntry::validate(); } SPIRVId Opn; }; template class SPIRVSubgroupAvcINTELType : public SPIRVTypeSubgroupAvcINTEL { public: // Complete constructor SPIRVSubgroupAvcINTELType(SPIRVModule *M, SPIRVId TheId) : SPIRVTypeSubgroupAvcINTEL(TheOpCode, M, TheId) {} // Incomplete constructor SPIRVSubgroupAvcINTELType() : SPIRVTypeSubgroupAvcINTEL(TheOpCode) {} }; #define _SPIRV_OP(x) \ typedef SPIRVSubgroupAvcINTELType SPIRVType##x##INTEL; _SPIRV_OP(AvcMcePayload) _SPIRV_OP(AvcImePayload) _SPIRV_OP(AvcRefPayload) _SPIRV_OP(AvcSicPayload) _SPIRV_OP(AvcMceResult) _SPIRV_OP(AvcImeResult) _SPIRV_OP(AvcImeResultSingleReferenceStreamout) _SPIRV_OP(AvcImeResultDualReferenceStreamout) _SPIRV_OP(AvcImeSingleReferenceStreamin) _SPIRV_OP(AvcImeDualReferenceStreamin) _SPIRV_OP(AvcRefResult) _SPIRV_OP(AvcSicResult) #undef _SPIRV_OP class SPIRVTypeTokenINTEL : public SPIRVType { public: // Complete constructor SPIRVTypeTokenINTEL(SPIRVModule *M, SPIRVId TheId) : SPIRVType(M, 2, internal::OpTypeTokenINTEL, TheId) {} // Incomplete constructor SPIRVTypeTokenINTEL() : SPIRVType(internal::OpTypeTokenINTEL) {} SPIRVCapVec getRequiredCapability() const override { return getVec(internal::CapabilityTokenTypeINTEL); } llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_token_type; } protected: _SPIRV_DEF_ENCDEC1(Id) }; class SPIRVTypeJointMatrixINTEL : public SPIRVType { SPIRVType *CompType; std::vector Args; public: const static Op OC = internal::OpTypeJointMatrixINTEL; const static SPIRVWord FixedWC = 3; // Complete constructor SPIRVTypeJointMatrixINTEL(SPIRVModule *M, SPIRVId TheId, SPIRVType *CompType, std::vector Args); // Incomplete constructor SPIRVTypeJointMatrixINTEL(); _SPIRV_DCL_ENCDEC llvm::Optional getRequiredExtension() const override { return ExtensionID::SPV_INTEL_joint_matrix; } SPIRVCapVec getRequiredCapability() const override { return {internal::CapabilityJointMatrixINTEL}; } void setWordCount(SPIRVWord WordCount) override { SPIRVType::setWordCount(WordCount); Args.resize(WordCount - FixedWC); } SPIRVType *getCompType() const { return CompType; } SPIRVValue *getRows() const { return Args[0]; } SPIRVValue *getColumns() const { return Args[1]; } SPIRVValue *getLayout() const { return Args[2]; } SPIRVValue *getScope() const { return Args[3]; } SPIRVValue *getUse() const { return Args.size() > 4 ? Args[4] : nullptr; } }; class SPIRVTypeCooperativeMatrixKHR : public SPIRVType { SPIRVType *CompType; std::vector Args; public: const static Op OC = OpTypeCooperativeMatrixKHR; const static SPIRVWord FixedWC = 7; // Incomplete constructor SPIRVTypeCooperativeMatrixKHR(SPIRVModule *M, SPIRVId TheId, SPIRVType *CompType, std::vector Args); // Incomplete constructor SPIRVTypeCooperativeMatrixKHR(); _SPIRV_DCL_ENCDEC llvm::Optional getRequiredExtension() const override { SPIRVType *Ty = this->getCompType(); if (Ty->isTypeInt() && static_cast(Ty)->getBitWidth() == 4) this->getModule()->addExtension(ExtensionID::SPV_INTEL_int4); return ExtensionID::SPV_KHR_cooperative_matrix; } SPIRVCapVec getRequiredCapability() const override { auto CV = getVec(CapabilityCooperativeMatrixKHR); if (CompType->isTypeFloat(16, FPEncodingBFloat16KHR)) CV.push_back(CapabilityBFloat16CooperativeMatrixKHR); else if (CompType->isTypeInt() && static_cast(CompType)->getBitWidth() == 4) CV.push_back(CapabilityInt4CooperativeMatrixINTEL); return CV; } SPIRVType *getCompType() const { return CompType; } SPIRVValue *getScope() const { return Args[0]; } SPIRVValue *getRows() const { return Args[1]; } SPIRVValue *getColumns() const { return Args[2]; } SPIRVValue *getUse() const { return Args[3]; } }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVTYPE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVUtil.h000066400000000000000000000276331506646577400225030ustar00rootroot00000000000000//===- SPIRVUtil.h - SPIR-V Utility Functions -------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines SPIR-V utility functions. /// //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVUTIL_H #define SPIRV_LIBSPIRV_SPIRVUTIL_H #include #define spv_ostream std::ostream #include "llvm/Support/raw_ostream.h" #include #include #include #include #include #include #include #include #include #include #include // MSVC supports "magic statics" since MSVS 2015. // For the previous version of MSVS we should guard // initialization of local static variables. #if defined(_MSC_VER) && (_MSC_VER < 1900) #include "llvm/Support/Mutex.h" #include "llvm/Support/MutexGuard.h" #endif // LLVM_MSC_PREREQ(1900) namespace SPIRV { #if defined(_MSC_VER) && (_MSC_VER < 1900) static llvm::sys::Mutex MapLock; #endif // LLVM_MSC_PREREQ(1900) #define SPIRV_DEF_NAMEMAP(Type, MapType) \ typedef SPIRVMap(MapType); \ inline MapType getNameMap(Type) { \ MapType MT; \ return MT; \ } constexpr unsigned MaxWordCount = UINT16_MAX; // A bi-way map template struct SPIRVMap { public: typedef Ty1 KeyTy; typedef Ty2 ValueTy; // Initialize map entries void init(); static Ty2 map(Ty1 Key) { Ty2 Val = {}; bool Found = find(Key, &Val); (void)Found; assert(Found && "Invalid key"); return Val; } static Ty1 rmap(Ty2 Key) { Ty1 Val = {}; bool Found = rfind(Key, &Val); (void)Found; assert(Found && "Invalid key"); return Val; } static const SPIRVMap &getMap() { #if defined(_MSC_VER) && (_MSC_VER < 1900) llvm::sys::ScopedLock mapGuard(MapLock); #endif // LLVM_MSC_PREREQ(1900) static const SPIRVMap Map(false); return Map; } static const SPIRVMap &getRMap() { #if defined(_MSC_VER) && (_MSC_VER < 1900) llvm::sys::ScopedLock mapGuard(MapLock); #endif // LLVM_MSC_PREREQ(1900) static const SPIRVMap Map(true); return Map; } static void foreach (std::function F) { for (auto &I : getMap().Map) F(I.first, I.second); } // For each key/value in the map executes function \p F. // If \p F returns false break the iteration. static void foreachConditional(std::function F) { for (auto &I : getMap().Map) { if (!F(I.first, I.second)) break; } } static bool find(Ty1 Key, Ty2 *Val = nullptr) { const SPIRVMap &Map = getMap(); typename MapTy::const_iterator Loc = Map.Map.find(Key); if (Loc == Map.Map.end()) return false; if (Val) *Val = Loc->second; return true; } static bool rfind(Ty2 Key, Ty1 *Val = nullptr) { const SPIRVMap &Map = getRMap(); typename RevMapTy::const_iterator Loc = Map.RevMap.find(Key); if (Loc == Map.RevMap.end()) return false; if (Val) *Val = Loc->second; return true; } SPIRVMap() : IsReverse(false) {} protected: SPIRVMap(bool Reverse) : IsReverse(Reverse) { init(); } typedef std::map MapTy; typedef std::map RevMapTy; void add(Ty1 V1, Ty2 V2) { if (IsReverse) { RevMap[V2] = V1; return; } Map[V1] = V2; } MapTy Map; RevMapTy RevMap; bool IsReverse; }; inline std::vector getVec(const std::string &S, char Delim) { std::vector Strs; std::stringstream SS(S); std::string Item; while (std::getline(SS, Item, Delim)) Strs.push_back(Item); return Strs; } inline std::unordered_set getUnordSet(const std::string &S, char Delim = ' ') { std::unordered_set Strs; std::stringstream SS(S); std::string Item; while (std::getline(SS, Item, Delim)) Strs.insert(Item); return Strs; } inline std::set getSet(const std::string &S, char Delim = ' ') { std::set Strs; std::stringstream SS(S); std::string Item; while (std::getline(SS, Item, Delim)) Strs.insert(Item); return Strs; } template VT map(KT Key) { return SPIRVMap::map(Key); } template KT rmap(VT V) { return SPIRVMap::rmap(V); } template std::unordered_set map(const std::unordered_set &KSet) { VT V; std::unordered_set VSet; for (auto &I : KSet) if (SPIRVMap::find(I, &V)) VSet.insert(V); return VSet; } template std::set map(const std::set &KSet) { VT V; std::set VSet; for (auto &I : KSet) if (SPIRVMap::find(I, &V)) VSet.insert(V); return VSet; } template std::unordered_set rmap(const std::unordered_set &KSet) { KT V; std::unordered_set VSet; for (auto &I : KSet) if (SPIRVMap::rfind(I, &V)) VSet.insert(V); return VSet; } template std::set rmap(const std::set &KSet) { KT V; std::set VSet; for (auto &I : KSet) if (SPIRVMap::rfind(I, &V)) VSet.insert(V); return VSet; } template std::set rmap(const std::map &KMap) { KT V; std::set VSet; for (auto &I : KMap) if (SPIRVMap::rfind(I.first, &V)) VSet.insert(V); return VSet; } template std::string getName(K Key) { std::string Name; if (SPIRVMap::find(Key, &Name)) return Name; return ""; } template bool getByName(const std::string &Name, K &Key) { return SPIRVMap::rfind(Name, &Key); } // Add a number as a string to a string template std::string concat(const std::string &S, const T &N) { std::stringstream Ss; Ss << S << N; return Ss.str(); } inline std::string concat(const std::string &S1, const std::string &S2, char Delim = ' ') { std::string S; if (S1.empty()) S = S2; else if (!S2.empty()) S = S1 + Delim + S2; return S; } inline std::string operator+(const std::string &S, int N) { return concat(S, N); } inline std::string operator+(const std::string &S, unsigned N) { return concat(S, N); } template std::string getStr(const T &C, char Delim = ' ') { std::stringstream SS; bool First = true; for (auto &I : C) { if (!First) SS << Delim; else First = false; SS << I; } return SS.str(); } template unsigned mapBitMask(unsigned BM) { unsigned Res = 0; MapTy::foreach ([&](typename MapTy::KeyTy K, typename MapTy::ValueTy V) { Res |= BM & (unsigned)K ? (unsigned)V : 0; }); return Res; } template unsigned rmapBitMask(unsigned BM) { unsigned Res = 0; MapTy::foreach ([&](typename MapTy::KeyTy K, typename MapTy::ValueTy V) { Res |= BM & (unsigned)V ? (unsigned)K : 0; }); return Res; } // Get the number of words used for encoding a string literal in SPIRV inline unsigned getSizeInWords(const std::string &Str) { assert(Str.length() / 4 + 1 <= std::numeric_limits::max()); return static_cast(Str.length() / 4 + 1); } inline std::string getString(std::vector::const_iterator Begin, std::vector::const_iterator End) { std::string Str = std::string(); for (auto I = Begin; I != End; ++I) { uint32_t Word = *I; for (unsigned J = 0u; J < 32u; J += 8u) { char Char = (char)((Word >> J) & 0xff); if (Char == '\0') return Str; Str += Char; } } return Str; } inline std::string getString(const std::vector &V) { return getString(V.cbegin(), V.cend()); } // if vector of Literals is expected to contain more than one Literal String inline std::vector getVecString(const std::vector &V) { std::vector Result; std::string Str; for (auto It = V.cbegin(); It < V.cend(); It += getSizeInWords(Str)) { Str.clear(); Str = getString(It, V.cend()); Result.push_back(Str); } return Result; } inline std::vector getVec(const std::string &Str) { std::vector V; auto StrSize = Str.size(); uint32_t CurrentWord = 0u; for (unsigned I = 0u; I < StrSize; ++I) { if (I % 4u == 0u && I != 0u) { V.push_back(CurrentWord); CurrentWord = 0u; } assert(Str[I] && "0 is not allowed in string"); CurrentWord += ((uint32_t)Str[I]) << ((I % 4u) * 8u); } if (CurrentWord != 0u) V.push_back(CurrentWord); if (StrSize % 4 == 0) V.push_back(0); return V; } template inline std::set getSet(T Op1) { std::set S; S.insert(Op1); return S; } template inline std::vector getVec(T Op1) { std::vector V; V.push_back(Op1); return V; } template inline std::vector getVec(T Op1, T Op2) { std::vector V; V.push_back(Op1); V.push_back(Op2); return V; } template inline std::vector getVec(T Op1, T Op2, T Op3) { std::vector V; V.push_back(Op1); V.push_back(Op2); V.push_back(Op3); return V; } template inline std::vector getVec(T Op1, const std::vector &Ops2) { std::vector V; V.push_back(Op1); V.insert(V.end(), Ops2.begin(), Ops2.end()); return V; } template typename MapTy::mapped_type getOrInsert(MapTy &Map, typename MapTy::key_type Key, FuncTy Func) { typename MapTy::iterator Loc = Map.find(Key); if (Loc != Map.end()) return Loc->second; typename MapTy::mapped_type NF = Func(); Map[Key] = NF; return NF; } template std::string toString(const T *Object) { if (Object == nullptr) return ""; std::string S; llvm::raw_string_ostream RSOS(S); Object->print(RSOS); RSOS.flush(); return S; } } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVUTIL_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVValue.cpp000066400000000000000000000143151506646577400231660ustar00rootroot00000000000000//===- SPIRVValue.cpp - Class to represent a SPIR-V Value -------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the values defined in SPIR-V spec with op codes. /// /// The name of the SPIR-V values follow the op code name in the spec. /// This is for readability and ease of using macro to handle types. // //===----------------------------------------------------------------------===// #include "SPIRVValue.h" #include "SPIRVEnum.h" #include "llvm/ADT/APInt.h" namespace SPIRV { void SPIRVValue::setAlignment(SPIRVWord A) { if (A == 0) { eraseDecorate(DecorationAlignment); return; } SPIRVWord PrevAlignment; if (hasAlignment(&PrevAlignment)) { // Do nothing if the Id already has an Alignment decoration, provided // it matches the new alignment. assert(A == PrevAlignment && "New alignment does not match existing alignment"); return; } addDecorate(new SPIRVDecorate(DecorationAlignment, this, A)); SPIRVDBG(spvdbgs() << "Set alignment " << A << " for obj " << Id << "\n") } bool SPIRVValue::hasAlignment(SPIRVWord *Result) const { return hasDecorate(DecorationAlignment, 0, Result); } bool SPIRVValue::isVolatile() const { return hasDecorate(DecorationVolatile); } void SPIRVValue::setVolatile(bool IsVolatile) { if (!IsVolatile) { eraseDecorate(DecorationVolatile); return; } addDecorate(new SPIRVDecorate(DecorationVolatile, this)); SPIRVDBG(spvdbgs() << "Set volatile " << " for obj " << Id << "\n") } bool SPIRVValue::hasNoSignedWrap() const { return hasDecorate(DecorationNoSignedWrap); } bool SPIRVValue::hasNoUnsignedWrap() const { return hasDecorate(DecorationNoUnsignedWrap); } void SPIRVValue::setFPFastMathMode(SPIRVWord M) { if (M == 0) { eraseDecorate(DecorationFPFastMathMode); return; } addDecorate(new SPIRVDecorate(DecorationFPFastMathMode, this, M)); SPIRVDBG(spvdbgs() << "Set fast math mode to " << M << " for obj " << Id << "\n") } template void SPIRVValue::setNoIntegerDecorationWrap(bool HasNoIntegerWrap) { if (!HasNoIntegerWrap) { eraseDecorate(NoIntegerWrapDecoration); return; } // NoSignedWrap and NoUnsignedWrap decorations are available only if it is // allowed to use SPIR-V 1.4 or if SPV_KHR_no_integer_wrap_decoration // extension is enabled #ifdef _SPIRVDBG const std::string InstStr = NoIntegerWrapDecoration == DecorationNoSignedWrap ? "nsw" : "nuw"; #endif // _SPIRVDBG if (Module->isAllowedToUseVersion(VersionNumber::SPIRV_1_4)) { Module->setMinSPIRVVersion(VersionNumber::SPIRV_1_4); addDecorate(new SPIRVDecorate(NoIntegerWrapDecoration, this)); SPIRVDBG(spvdbgs() << "Set " << InstStr << " for obj " << Id << "\n") } else if (Module->isAllowedToUseExtension( ExtensionID::SPV_KHR_no_integer_wrap_decoration)) { Module->addExtension(ExtensionID::SPV_KHR_no_integer_wrap_decoration); addDecorate(new SPIRVDecorate(NoIntegerWrapDecoration, this)); SPIRVDBG(spvdbgs() << "Set " << InstStr << " for obj " << Id << "\n") } else { SPIRVDBG(spvdbgs() << "Skip setting " << InstStr << " for obj " << Id << "\n") } } template void SPIRVValue::setNoIntegerDecorationWrap(bool); template void SPIRVValue::setNoIntegerDecorationWrap(bool); template void SPIRVConstantBase::setWords(const uint64_t *TheValue) { assert(TheValue && "Nullptr value"); recalculateWordCount(); validate(); Words.resize(NumWords); for (size_t I = 0; I != NumWords / 2; ++I) { Words[I * 2] = static_cast(TheValue[I]) & SPIRVWORD_MAX; Words[I * 2 + 1] = static_cast((TheValue[I] >> SpirvWordBitWidth)) & SPIRVWORD_MAX; } if (NumWords % 2) Words.back() = static_cast(TheValue[NumWords / 2]) & SPIRVWORD_MAX; } // Complete constructor for AP integer constant template SPIRVConstantBase::SPIRVConstantBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, const llvm::APInt &TheValue) : SPIRVValue(M, 0, OC, TheType, TheId) { setWords(TheValue.getRawData()); } // To solve errors about undefined reference to template class methods // definitions. template class SPIRVConstantBase; template class SPIRVConstantBase; } // namespace SPIRV SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/SPIRVValue.h000066400000000000000000000365601506646577400226410ustar00rootroot00000000000000//===- SPIRVValue.h - Class to represent a SPIR-V Value ---------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file defines the values defined in SPIR-V spec with op codes. /// /// The name of the SPIR-V values follow the op code name in the spec. /// This is for readability and ease of using macro to handle types. // //===----------------------------------------------------------------------===// #ifndef SPIRV_LIBSPIRV_SPIRVVALUE_H #define SPIRV_LIBSPIRV_SPIRVVALUE_H #include "SPIRVDecorate.h" #include "SPIRVEntry.h" #include "SPIRVType.h" namespace llvm { class APInt; } // namespace llvm #include namespace SPIRV { class SPIRVValue : public SPIRVEntry { public: // Complete constructor for value with id and type SPIRVValue(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode, SPIRVType *TheType, SPIRVId TheId) : SPIRVEntry(M, TheWordCount, TheOpCode, TheId), Type(TheType) { validate(); } // Complete constructor for value with type but without id SPIRVValue(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode, SPIRVType *TheType) : SPIRVEntry(M, TheWordCount, TheOpCode), Type(TheType) { setHasNoId(); SPIRVValue::validate(); } // Complete constructor for value with id but without type SPIRVValue(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode, SPIRVId TheId) : SPIRVEntry(M, TheWordCount, TheOpCode, TheId), Type(NULL) { setHasNoType(); SPIRVValue::validate(); } // Complete constructor for value without id and type SPIRVValue(SPIRVModule *M, unsigned TheWordCount, Op TheOpCode) : SPIRVEntry(M, TheWordCount, TheOpCode), Type(NULL) { setHasNoId(); setHasNoType(); SPIRVValue::validate(); } // Incomplete constructor SPIRVValue(Op TheOpCode) : SPIRVEntry(TheOpCode), Type(NULL) {} bool hasType() const { return !(Attrib & SPIRVEA_NOTYPE); } SPIRVType *getType() const { assert(hasType() && "value has no type"); return Type; } bool isVolatile() const; bool hasAlignment(SPIRVWord *Result = 0) const; bool hasNoSignedWrap() const; bool hasNoUnsignedWrap() const; void setAlignment(SPIRVWord); void setVolatile(bool IsVolatile); template void setNoIntegerDecorationWrap(bool HasNoIntegerWrap); void setFPFastMathMode(SPIRVWord FPFastMathMode); void validate() const override { SPIRVEntry::validate(); assert((!hasType() || Type) && "Invalid type"); } void setType(SPIRVType *Ty) { Type = Ty; assert(!Ty || !Ty->isTypeVoid() || OpCode == OpFunction); if (Ty && (!Ty->isTypeVoid() || OpCode == OpFunction)) setHasType(); else setHasNoType(); } SPIRVCapVec getRequiredCapability() const override { SPIRVCapVec CV; if (!hasType()) return CV; return Type->getRequiredCapability(); } llvm::Optional getRequiredExtension() const override { llvm::Optional EV; if (!hasType()) return EV; EV = Type->getRequiredExtension(); assert(Module && (!EV.hasValue() || Module->isAllowedToUseExtension(EV.getValue()))); return EV; } protected: void setHasNoType() { Attrib |= SPIRVEA_NOTYPE; } void setHasType() { Attrib &= ~SPIRVEA_NOTYPE; } SPIRVType *Type; // Value Type }; template class SPIRVConstantBase : public SPIRVValue { public: // Complete constructor for integer constant SPIRVConstantBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, uint64_t TheValue) : SPIRVValue(M, 0, OC, TheType, TheId) { setWords(&TheValue); } // Incomplete constructor for AP integer constant SPIRVConstantBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, const llvm::APInt &TheValue); // Complete constructor for float constant SPIRVConstantBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, float TheValue) : SPIRVValue(M, 0, OC, TheType, TheId) { setWords(reinterpret_cast(&TheValue)); } // Complete constructor for double constant SPIRVConstantBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, double TheValue) : SPIRVValue(M, 0, OC, TheType, TheId) { setWords(reinterpret_cast(&TheValue)); } // Incomplete constructor SPIRVConstantBase() : SPIRVValue(OC), NumWords(0) {} uint64_t getZExtIntValue() const { return getValue(); } float getFloatValue() const { return getValue(); } double getDoubleValue() const { return getValue(); } unsigned getNumWords() const { return NumWords; } const std::vector &getSPIRVWords() { return Words; } protected: constexpr static SPIRVWord FixedWC = 3; // Common method for getting values of size less or equal to 64 bits. template T getValue() const { constexpr auto ValueSize = static_cast(sizeof(T)); assert((ValueSize <= 8) && "Incorrect result type of requested value"); T TheValue{}; unsigned CopyBytes = std::min(ValueSize, NumWords * SpirvWordSize); std::memcpy(&TheValue, Words.data(), CopyBytes); return TheValue; } void setWords(const uint64_t *TheValue); void recalculateWordCount() { NumWords = (Type->getBitWidth() + SpirvWordBitWidth - 1) / SpirvWordBitWidth; WordCount = FixedWC + NumWords; } void validate() const override { SPIRVValue::validate(); assert(NumWords >= 1 && "Invalid constant size"); } void encode(spv_ostream &O) const override { getEncoder(O) << Type << Id; for (const auto &Word : Words) getEncoder(O) << Word; } void setWordCount(SPIRVWord WordCount) override { SPIRVValue::setWordCount(WordCount); NumWords = WordCount - FixedWC; } void decode(std::istream &I) override { getDecoder(I) >> Type >> Id; Words.resize(NumWords); for (auto &Word : Words) getDecoder(I) >> Word; } unsigned NumWords; private: std::vector Words; }; using SPIRVConstant = SPIRVConstantBase; using SPIRVSpecConstant = SPIRVConstantBase; template class SPIRVConstantEmpty : public SPIRVValue { public: // Complete constructor SPIRVConstantEmpty(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId) : SPIRVValue(M, 3, OC, TheType, TheId) { validate(); } // Incomplete constructor SPIRVConstantEmpty() : SPIRVValue(OC) {} protected: void validate() const override { SPIRVValue::validate(); } _SPIRV_DEF_ENCDEC2(Type, Id) }; template class SPIRVConstantBool : public SPIRVConstantEmpty { public: // Complete constructor SPIRVConstantBool(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId) : SPIRVConstantEmpty(M, TheType, TheId) {} // Incomplete constructor SPIRVConstantBool() {} protected: void validate() const override { SPIRVConstantEmpty::validate(); assert(this->Type->isTypeBool() && "Invalid type"); } }; typedef SPIRVConstantBool SPIRVConstantTrue; typedef SPIRVConstantBool SPIRVConstantFalse; typedef SPIRVConstantBool SPIRVSpecConstantTrue; typedef SPIRVConstantBool SPIRVSpecConstantFalse; class SPIRVConstantNull : public SPIRVConstantEmpty { public: // Complete constructor SPIRVConstantNull(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId) : SPIRVConstantEmpty(M, TheType, TheId) { validate(); } // Incomplete constructor SPIRVConstantNull() {} protected: void validate() const override { SPIRVConstantEmpty::validate(); assert((Type->isTypeBool() || Type->isTypeInt() || Type->isTypeFloat() || Type->isTypeComposite() || Type->isTypeOpaque() || Type->isTypeEvent() || Type->isTypePointer() || Type->isTypeReserveId() || Type->isTypeDeviceEvent() || (Type->isTypeSubgroupAvcINTEL() && !Type->isTypeSubgroupAvcMceINTEL())) && "Invalid type"); } }; class SPIRVUndef : public SPIRVConstantEmpty { public: // Complete constructor SPIRVUndef(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId) : SPIRVConstantEmpty(M, TheType, TheId) { validate(); } // Incomplete constructor SPIRVUndef() {} protected: void validate() const override { SPIRVConstantEmpty::validate(); } }; template class SPIRVConstantCompositeBase : public SPIRVValue { public: // There are always 3 words in this instruction except constituents: // 1) WordCount + OpCode // 2) Result type // 3) Result Id constexpr static SPIRVWord FixedWC = 3; using ContinuedInstType = typename InstToContinued::Type; // Complete constructor for composite constant SPIRVConstantCompositeBase(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, const std::vector TheElements) : SPIRVValue(M, TheElements.size() + FixedWC, OC, TheType, TheId) { Elements = getIds(TheElements); validate(); } // Incomplete constructor SPIRVConstantCompositeBase() : SPIRVValue(OC) {} std::vector getElements() const { return getValues(Elements); } // TODO: Should we attach operands of continued instructions as well? std::vector getNonLiteralOperands() const override { std::vector Elements = getElements(); return std::vector(Elements.begin(), Elements.end()); } std::vector getContinuedInstructions() { return ContinuedInstructions; } void addContinuedInstruction(ContinuedInstType Inst) { ContinuedInstructions.push_back(Inst); } void encodeChildren(spv_ostream &O) const override { O << SPIRVNL(); for (auto &I : ContinuedInstructions) O << *I; } protected: void validate() const override { SPIRVValue::validate(); for (auto &I : Elements) getValue(I)->validate(); } void setWordCount(SPIRVWord WordCount) override { SPIRVEntry::setWordCount(WordCount); Elements.resize(WordCount - FixedWC); } void encode(spv_ostream &O) const override { getEncoder(O) << Type << Id << Elements; } void decode(std::istream &I) override { SPIRVDecoder Decoder = getDecoder(I); Decoder >> Type >> Id >> Elements; for (SPIRVEntry *E : Decoder.getContinuedInstructions(ContinuedOpCode)) { addContinuedInstruction(static_cast(E)); } } std::vector Elements; std::vector ContinuedInstructions; const spv::Op ContinuedOpCode = InstToContinued::OpCode; }; using SPIRVConstantComposite = SPIRVConstantCompositeBase; using SPIRVSpecConstantComposite = SPIRVConstantCompositeBase; class SPIRVConstantSampler : public SPIRVValue { public: const static Op OC = OpConstantSampler; const static SPIRVWord WC = 6; // Complete constructor SPIRVConstantSampler(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, SPIRVWord TheAddrMode, SPIRVWord TheNormalized, SPIRVWord TheFilterMode) : SPIRVValue(M, WC, OC, TheType, TheId), AddrMode(TheAddrMode), Normalized(TheNormalized), FilterMode(TheFilterMode) { validate(); } // Incomplete constructor SPIRVConstantSampler() : SPIRVValue(OC), AddrMode(SPIRVSAM_Invalid), Normalized(SPIRVWORD_MAX), FilterMode(SPIRVSFM_Invalid) {} SPIRVWord getAddrMode() const { return AddrMode; } SPIRVWord getFilterMode() const { return FilterMode; } SPIRVWord getNormalized() const { return Normalized; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityLiteralSampler); } protected: SPIRVWord AddrMode; SPIRVWord Normalized; SPIRVWord FilterMode; void validate() const override { SPIRVValue::validate(); assert(OpCode == OC); assert(WordCount == WC); assert(Type->isTypeSampler()); } _SPIRV_DEF_ENCDEC5(Type, Id, AddrMode, Normalized, FilterMode) }; class SPIRVConstantPipeStorage : public SPIRVValue { public: const static Op OC = OpConstantPipeStorage; const static SPIRVWord WC = 6; // Complete constructor SPIRVConstantPipeStorage(SPIRVModule *M, SPIRVType *TheType, SPIRVId TheId, SPIRVWord ThePacketSize, SPIRVWord ThePacketAlign, SPIRVWord TheCapacity) : SPIRVValue(M, WC, OC, TheType, TheId), PacketSize(ThePacketSize), PacketAlign(ThePacketAlign), Capacity(TheCapacity) { validate(); } // Incomplete constructor SPIRVConstantPipeStorage() : SPIRVValue(OC), PacketSize(0), PacketAlign(0), Capacity(0) {} SPIRVWord getPacketSize() const { return PacketSize; } SPIRVWord getPacketAlign() const { return PacketAlign; } SPIRVWord getCapacity() const { return Capacity; } SPIRVCapVec getRequiredCapability() const override { return getVec(CapabilityPipes, CapabilityPipeStorage); } protected: SPIRVWord PacketSize; SPIRVWord PacketAlign; SPIRVWord Capacity; void validate() const override { SPIRVValue::validate(); assert(OpCode == OC); assert(WordCount == WC); assert(Type->isTypePipeStorage()); } _SPIRV_DEF_ENCDEC5(Type, Id, PacketSize, PacketAlign, Capacity) }; class SPIRVForward : public SPIRVValue, public SPIRVComponentExecutionModes { public: const static Op OC = internal::OpForward; // Complete constructor SPIRVForward(SPIRVModule *TheModule, SPIRVType *TheTy, SPIRVId TheId) : SPIRVValue(TheModule, 0, OC, TheId) { if (TheTy) setType(TheTy); } SPIRVForward() : SPIRVValue(OC) { assert(0 && "should never be called"); } _SPIRV_DEF_ENCDEC1(Id) friend class SPIRVFunction; protected: void validate() const override {} }; } // namespace SPIRV #endif // SPIRV_LIBSPIRV_SPIRVVALUE_H SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/libSPIRV.h000066400000000000000000000044271506646577400223300ustar00rootroot00000000000000//===- libSPIRV.h - SPIR-V Header files -------------------------*- C++ -*-===// // // The LLVM/SPIRV Translator // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // // Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // Redistributions in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimers in the documentation // and/or other materials provided with the distribution. // Neither the names of Advanced Micro Devices, Inc., nor the names of its // contributors may be used to endorse or promote products derived from this // Software without specific prior written permission. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH // THE SOFTWARE. // //===----------------------------------------------------------------------===// /// \file /// /// This file includes all SPIRV header files. /// //===----------------------------------------------------------------------===// #ifndef LIBSPIRV_H_ #define LIBSPIRV_H_ #include "SPIRVBasicBlock.h" #include "SPIRVEntry.h" #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVModule.h" #include "SPIRVOpCode.h" #include "SPIRVType.h" #include "SPIRVValue.h" #endif // LIBSPIRV_H_ SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/libSPIRV/spirv_internal.hpp000066400000000000000000000247651506646577400243440ustar00rootroot00000000000000// Copyright (c) 2020 The Khronos Group Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and/or associated documentation files (the "Materials"), // to deal in the Materials without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Materials, and to permit persons to whom the // Materials are furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Materials. // // THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM,OUT OF OR IN CONNECTION WITH THE MATERIALS OR THE USE OR OTHER DEALINGS // IN THE MATERIALS. // The header is for SPIR-V to LLVM IR internal definitions, that are not a part // of Khronos SPIR-V specification. #include "spirv/unified1/spirv.hpp" #ifndef spirv_internal_HPP #define spirv_internal_HPP namespace spv { namespace internal { enum InternalSourceLanguageNonSemanticDI { ISourceLanguagePython = 200, ISourceLanguageJulia = 201, ISourceLanguageRust = 202, ISourceLanguageD = 203, ISourceLanguageFortran77 = 204, ISourceLanguageFortran90 = 205, ISourceLanguageFortran95 = 206, ISourceLanguageFortran2003 = 207, ISourceLanguageFortran2008 = 208, ISourceLanguageFortran2018 = 209, ISourceLanguageC = 210, ISourceLanguageC99 = 211, ISourceLanguageC11 = 212, ISourceLanguageC17 = 213, ISourceLanguageCPP = 214, ISourceLanguageCPP03 = 215, ISourceLanguageCPP11 = 216, ISourceLanguageCPP14 = 217, ISourceLanguageCPP17 = 218, ISourceLanguageCPP20 = 219, }; enum InternalLinkageType { ILTPrev = LinkageTypeMax - 2, ILTInternal }; enum InternalOp { IOpTypeTokenINTEL = 6113, IOpConvertFToBF16INTEL = 6116, IOpConvertBF16ToFINTEL = 6117, IOpTypeJointMatrixINTEL = 6119, IOpJointMatrixLoadINTEL = 6120, IOpJointMatrixStoreINTEL = 6121, IOpJointMatrixMadINTEL = 6122, IOpJointMatrixSUMadINTEL = 6128, IOpJointMatrixUSMadINTEL = 6129, IOpJointMatrixUUMadINTEL = 6130, IOpCooperativeMatrixLoadCheckedINTEL = 6193, IOpCooperativeMatrixStoreCheckedINTEL = 6194, IOpCooperativeMatrixConstructCheckedINTEL = 6195, IOpJointMatrixWorkItemLengthINTEL = 6410, IOpComplexFMulINTEL = 6415, IOpComplexFDivINTEL = 6416, IOpRoundFToTF32INTEL = 6426, IOpMaskedGatherINTEL = 6428, IOpMaskedScatterINTEL = 6429, IOpJointMatrixGetElementCoordINTEL = 6440, IOpCooperativeMatrixPrefetchINTEL = 6449, IOpConvertHandleToImageINTEL = 6529, IOpConvertHandleToSamplerINTEL = 6530, IOpConvertHandleToSampledImageINTEL = 6531, IOpPrev = OpMax - 2, IOpForward }; enum InternalDecoration { IDecMathOpDSPModeINTEL = 5909, IDecInitiationIntervalINTEL = 5917, IDecMaxConcurrencyINTEL = 5918, IDecPipelineEnableINTEL = 5919, IDecRuntimeAlignedINTEL = 5940, IDecCallableFunctionINTEL = 6087, IDecHostAccessINTEL = 6147, IDecInitModeINTEL = 6148, IDecImplementInCSRINTEL = 6149, IDecArgumentAttributeINTEL = 6409, }; enum InternalCapability { ICapFPGADSPControlINTEL = 5908, ICapFPGAInvocationPipeliningAttributesINTEL = 5916, ICapRuntimeAlignedAttributeINTEL = 5939, ICapTokenTypeINTEL = 6112, ICapBfloat16ConversionINTEL = 6115, ICapabilityJointMatrixINTEL = 6118, ICapabilityHWThreadQueryINTEL = 6134, ICapGlobalVariableDecorationsINTEL = 6146, ICapabilityCooperativeMatrixCheckedInstructionsINTEL = 6192, ICapabilityBFloat16ArithmeticINTEL = 6226, ICapabilityAtomicBFloat16AddINTEL = 6255, ICapabilityAtomicBFloat16MinMaxINTEL = 6256, ICapabilityCooperativeMatrixPrefetchINTEL = 6411, ICapabilityComplexFloatMulDivINTEL = 6414, ICapabilityTensorFloat32RoundingINTEL = 6425, ICapabilityMaskedGatherScatterINTEL = 6427, ICapabilityJointMatrixWIInstructionsINTEL = 6435, ICapabilityCacheControlsINTEL = 6441, ICapabilityBindlessImagesINTEL = 6528 }; enum InternalExecutionMode {}; enum InternalLoopControlMask { ILoopControlLoopCountINTELMask = 0x1000000 }; constexpr LinkageType LinkageTypeInternal = static_cast(ILTInternal); enum InternalJointMatrixLayout { RowMajor = 0, ColumnMajor = 1, PackedA = 2, PackedB = 3 }; enum InternalJointMatrixUse { MatrixA = 0, MatrixB = 1, Accumulator = 2 }; enum InternalBuiltIn { IBuiltInSubDeviceIDINTEL = 6135, IBuiltInGlobalHWThreadIDINTEL = 6136, }; #define _SPIRV_OP(x, y) constexpr x x##y = static_cast(I##x##y); _SPIRV_OP(Capability, JointMatrixINTEL) _SPIRV_OP(Capability, JointMatrixWIInstructionsINTEL) _SPIRV_OP(Op, TypeJointMatrixINTEL) _SPIRV_OP(Op, JointMatrixLoadINTEL) _SPIRV_OP(Op, JointMatrixStoreINTEL) _SPIRV_OP(Op, JointMatrixMadINTEL) _SPIRV_OP(Op, JointMatrixSUMadINTEL) _SPIRV_OP(Op, JointMatrixUSMadINTEL) _SPIRV_OP(Op, JointMatrixUUMadINTEL) _SPIRV_OP(Op, JointMatrixWorkItemLengthINTEL) _SPIRV_OP(Op, JointMatrixGetElementCoordINTEL) _SPIRV_OP(Capability, CooperativeMatrixPrefetchINTEL) _SPIRV_OP(Op, CooperativeMatrixPrefetchINTEL) _SPIRV_OP(Capability, CooperativeMatrixCheckedInstructionsINTEL) _SPIRV_OP(Op, CooperativeMatrixLoadCheckedINTEL) _SPIRV_OP(Op, CooperativeMatrixStoreCheckedINTEL) _SPIRV_OP(Op, CooperativeMatrixConstructCheckedINTEL) _SPIRV_OP(Capability, HWThreadQueryINTEL) _SPIRV_OP(BuiltIn, SubDeviceIDINTEL) _SPIRV_OP(BuiltIn, GlobalHWThreadIDINTEL) _SPIRV_OP(Capability, ComplexFloatMulDivINTEL) _SPIRV_OP(Op, ComplexFMulINTEL) _SPIRV_OP(Op, ComplexFDivINTEL) _SPIRV_OP(Capability, MaskedGatherScatterINTEL) _SPIRV_OP(Op, MaskedGatherINTEL) _SPIRV_OP(Op, MaskedScatterINTEL) _SPIRV_OP(Capability, TensorFloat32RoundingINTEL) _SPIRV_OP(Op, RoundFToTF32INTEL) _SPIRV_OP(Capability, CacheControlsINTEL) _SPIRV_OP(Capability, BindlessImagesINTEL) _SPIRV_OP(Op, ConvertHandleToImageINTEL) _SPIRV_OP(Op, ConvertHandleToSamplerINTEL) _SPIRV_OP(Op, ConvertHandleToSampledImageINTEL) _SPIRV_OP(Capability, AtomicBFloat16AddINTEL) _SPIRV_OP(Capability, AtomicBFloat16MinMaxINTEL) #undef _SPIRV_OP constexpr SourceLanguage SourceLanguagePython = static_cast(ISourceLanguagePython); constexpr SourceLanguage SourceLanguageJulia = static_cast(ISourceLanguageJulia); constexpr SourceLanguage SourceLanguageRust = static_cast(ISourceLanguageRust); constexpr SourceLanguage SourceLanguageD = static_cast(ISourceLanguageD); constexpr SourceLanguage SourceLanguageFortran77 = static_cast(ISourceLanguageFortran77); constexpr SourceLanguage SourceLanguageFortran90 = static_cast(ISourceLanguageFortran90); constexpr SourceLanguage SourceLanguageFortran95 = static_cast(ISourceLanguageFortran95); constexpr SourceLanguage SourceLanguageFortran2003 = static_cast(ISourceLanguageFortran2003); constexpr SourceLanguage SourceLanguageFortran2008 = static_cast(ISourceLanguageFortran2008); constexpr SourceLanguage SourceLanguageFortran2018 = static_cast(ISourceLanguageFortran2018); constexpr SourceLanguage SourceLanguageC = static_cast(ISourceLanguageC); constexpr SourceLanguage SourceLanguageC99 = static_cast(ISourceLanguageC99); constexpr SourceLanguage SourceLanguageC11 = static_cast(ISourceLanguageC11); constexpr SourceLanguage SourceLanguageC17 = static_cast(ISourceLanguageC17); constexpr SourceLanguage SourceLanguageCPP = static_cast(ISourceLanguageCPP); constexpr SourceLanguage SourceLanguageCPP03 = static_cast(ISourceLanguageCPP03); constexpr SourceLanguage SourceLanguageCPP11 = static_cast(ISourceLanguageCPP11); constexpr SourceLanguage SourceLanguageCPP14 = static_cast(ISourceLanguageCPP14); constexpr SourceLanguage SourceLanguageCPP17 = static_cast(ISourceLanguageCPP17); constexpr SourceLanguage SourceLanguageCPP20 = static_cast(ISourceLanguageCPP20); constexpr Op OpForward = static_cast(IOpForward); constexpr Op OpTypeTokenINTEL = static_cast(IOpTypeTokenINTEL); constexpr Op OpConvertFToBF16INTEL = static_cast(IOpConvertFToBF16INTEL); constexpr Op OpConvertBF16ToFINTEL = static_cast(IOpConvertBF16ToFINTEL); constexpr Decoration DecorationInitiationIntervalINTEL = static_cast(IDecInitiationIntervalINTEL); constexpr Decoration DecorationMaxConcurrencyINTEL = static_cast(IDecMaxConcurrencyINTEL); constexpr Decoration DecorationPipelineEnableINTEL = static_cast(IDecPipelineEnableINTEL); constexpr Decoration DecorationRuntimeAlignedINTEL = static_cast(IDecRuntimeAlignedINTEL); constexpr Decoration DecorationHostAccessINTEL = static_cast(IDecHostAccessINTEL); constexpr Decoration DecorationInitModeINTEL = static_cast(IDecInitModeINTEL); constexpr Decoration DecorationImplementInCSRINTEL = static_cast(IDecImplementInCSRINTEL); constexpr Decoration DecorationArgumentAttributeINTEL = static_cast(IDecArgumentAttributeINTEL); constexpr Capability CapabilityFPGADSPControlINTEL = static_cast(ICapFPGADSPControlINTEL); constexpr Capability CapabilityFPGAInvocationPipeliningAttributesINTEL = static_cast(ICapFPGAInvocationPipeliningAttributesINTEL); constexpr Capability CapabilityTokenTypeINTEL = static_cast(ICapTokenTypeINTEL); constexpr Capability CapabilityRuntimeAlignedAttributeINTEL = static_cast(ICapRuntimeAlignedAttributeINTEL); constexpr Capability CapabilityBfloat16ConversionINTEL = static_cast(ICapBfloat16ConversionINTEL); constexpr Capability CapabilityGlobalVariableDecorationsINTEL = static_cast(ICapGlobalVariableDecorationsINTEL); constexpr Capability CapabilityBFloat16ArithmeticINTEL = static_cast(ICapabilityBFloat16ArithmeticINTEL); constexpr Decoration DecorationMathOpDSPModeINTEL = static_cast(IDecMathOpDSPModeINTEL); constexpr LoopControlMask LoopControlLoopCountINTELMask = static_cast(ILoopControlLoopCountINTELMask); } // namespace internal } // namespace spv #endif // #ifndef spirv_internal_HPP SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/000077500000000000000000000000001506646577400206475ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/000077500000000000000000000000001506646577400217675ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/inc/000077500000000000000000000000001506646577400225405ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/inc/spirv.h000066400000000000000000000024341506646577400240570ustar00rootroot00000000000000__attribute__((overloadable, always_inline)) int __spirv_ImageQuerySize(image1d_buffer_t img); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySize(image1d_array_t img); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySize(image2d_t img); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySize(image2d_depth_t img); __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySize(image2d_array_t img); __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySize(image2d_array_depth_t img); __attribute__((overloadable, always_inline)) int __spirv_ImageQuerySizeLod(image1d_t img, int lod); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySizeLod(image1d_array_t img, int lod); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySizeLod(image2d_t img, int lod); __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySizeLod(image2d_depth_t img, int lod); __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySizeLod(image2d_array_t img, int lod); __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySizeLod(image2d_array_depth_t img, int lod); __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySizeLod(image3d_t img, int lod); SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/inc/spirv_convert.h000066400000000000000000016655731506646577400256430ustar00rootroot00000000000000__attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtz(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rte(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtp(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtn(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtz(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rte(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtp(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtn(half); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtz(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rte(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtp(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtn(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtz(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rte(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtp(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtn(float); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtz(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rte(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtp(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_rtn(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtz(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rte(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtp(double); __attribute__((overloadable)) uchar __spirv_ConvertFToU_Ruchar_sat_rtn(double); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtz(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rte(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtp(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtn(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtz(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rte(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtp(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtn(half2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtz(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rte(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtp(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtn(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtz(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rte(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtp(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtn(float2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtz(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rte(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtp(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_rtn(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtz(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rte(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtp(double2); __attribute__((overloadable)) uchar2 __spirv_ConvertFToU_Ruchar2_sat_rtn(double2); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtz(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rte(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtp(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtn(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtz(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rte(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtp(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtn(half3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtz(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rte(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtp(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtn(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtz(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rte(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtp(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtn(float3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtz(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rte(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtp(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_rtn(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtz(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rte(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtp(double3); __attribute__((overloadable)) uchar3 __spirv_ConvertFToU_Ruchar3_sat_rtn(double3); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtz(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rte(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtp(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtn(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtz(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rte(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtp(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtn(half4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtz(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rte(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtp(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtn(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtz(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rte(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtp(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtn(float4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtz(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rte(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtp(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_rtn(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtz(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rte(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtp(double4); __attribute__((overloadable)) uchar4 __spirv_ConvertFToU_Ruchar4_sat_rtn(double4); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtz(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rte(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtp(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtn(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtz(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rte(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtp(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtn(half8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtz(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rte(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtp(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtn(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtz(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rte(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtp(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtn(float8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtz(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rte(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtp(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_rtn(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtz(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rte(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtp(double8); __attribute__((overloadable)) uchar8 __spirv_ConvertFToU_Ruchar8_sat_rtn(double8); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtz(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rte(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtp(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtn(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtz(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rte(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtp(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtn(half16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtz(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rte(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtp(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtn(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtz(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rte(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtp(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtn(float16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtz(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rte(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtp(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_rtn(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtz(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rte(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtp(double16); __attribute__((overloadable)) uchar16 __spirv_ConvertFToU_Ruchar16_sat_rtn(double16); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtz(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rte(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtp(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtn(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtz(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rte(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtp(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtn(half); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtz(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rte(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtp(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtn(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtz(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rte(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtp(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtn(float); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtz(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rte(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtp(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_rtn(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtz(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rte(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtp(double); __attribute__((overloadable)) ushort __spirv_ConvertFToU_Rushort_sat_rtn(double); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtz(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rte(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtp(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtn(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtz(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rte(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtp(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtn(half2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtz(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rte(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtp(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtn(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtz(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rte(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtp(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtn(float2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtz(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rte(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtp(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_rtn(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtz(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rte(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtp(double2); __attribute__((overloadable)) ushort2 __spirv_ConvertFToU_Rushort2_sat_rtn(double2); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtz(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rte(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtp(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtn(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtz(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rte(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtp(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtn(half3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtz(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rte(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtp(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtn(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtz(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rte(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtp(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtn(float3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtz(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rte(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtp(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_rtn(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtz(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rte(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtp(double3); __attribute__((overloadable)) ushort3 __spirv_ConvertFToU_Rushort3_sat_rtn(double3); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtz(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rte(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtp(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtn(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtz(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rte(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtp(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtn(half4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtz(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rte(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtp(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtn(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtz(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rte(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtp(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtn(float4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtz(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rte(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtp(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_rtn(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtz(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rte(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtp(double4); __attribute__((overloadable)) ushort4 __spirv_ConvertFToU_Rushort4_sat_rtn(double4); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtz(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rte(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtp(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtn(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtz(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rte(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtp(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtn(half8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtz(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rte(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtp(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtn(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtz(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rte(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtp(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtn(float8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtz(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rte(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtp(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_rtn(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtz(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rte(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtp(double8); __attribute__((overloadable)) ushort8 __spirv_ConvertFToU_Rushort8_sat_rtn(double8); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtz(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rte(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtp(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtn(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtz(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rte(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtp(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtn(half16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtz(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rte(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtp(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtn(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtz(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rte(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtp(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtn(float16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtz(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rte(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtp(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_rtn(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtz(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rte(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtp(double16); __attribute__((overloadable)) ushort16 __spirv_ConvertFToU_Rushort16_sat_rtn(double16); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtz(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rte(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtp(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtn(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtz(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rte(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtp(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtn(half); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtz(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rte(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtp(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtn(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtz(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rte(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtp(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtn(float); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtz(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rte(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtp(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_rtn(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtz(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rte(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtp(double); __attribute__((overloadable)) uint __spirv_ConvertFToU_Ruint_sat_rtn(double); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtz(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rte(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtp(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtn(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtz(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rte(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtp(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtn(half2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtz(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rte(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtp(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtn(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtz(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rte(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtp(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtn(float2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtz(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rte(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtp(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_rtn(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtz(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rte(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtp(double2); __attribute__((overloadable)) uint2 __spirv_ConvertFToU_Ruint2_sat_rtn(double2); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtz(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rte(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtp(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtn(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtz(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rte(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtp(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtn(half3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtz(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rte(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtp(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtn(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtz(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rte(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtp(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtn(float3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtz(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rte(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtp(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_rtn(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtz(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rte(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtp(double3); __attribute__((overloadable)) uint3 __spirv_ConvertFToU_Ruint3_sat_rtn(double3); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtz(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rte(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtp(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtn(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtz(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rte(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtp(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtn(half4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtz(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rte(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtp(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtn(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtz(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rte(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtp(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtn(float4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtz(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rte(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtp(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_rtn(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtz(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rte(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtp(double4); __attribute__((overloadable)) uint4 __spirv_ConvertFToU_Ruint4_sat_rtn(double4); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtz(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rte(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtp(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtn(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtz(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rte(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtp(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtn(half8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtz(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rte(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtp(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtn(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtz(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rte(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtp(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtn(float8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtz(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rte(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtp(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_rtn(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtz(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rte(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtp(double8); __attribute__((overloadable)) uint8 __spirv_ConvertFToU_Ruint8_sat_rtn(double8); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtz(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rte(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtp(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtn(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtz(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rte(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtp(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtn(half16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtz(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rte(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtp(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtn(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtz(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rte(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtp(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtn(float16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtz(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rte(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtp(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_rtn(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtz(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rte(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtp(double16); __attribute__((overloadable)) uint16 __spirv_ConvertFToU_Ruint16_sat_rtn(double16); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtz(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rte(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtp(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtn(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtz(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rte(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtp(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtn(half); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtz(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rte(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtp(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtn(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtz(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rte(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtp(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtn(float); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtz(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rte(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtp(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_rtn(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtz(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rte(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtp(double); __attribute__((overloadable)) ulong __spirv_ConvertFToU_Rulong_sat_rtn(double); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtz(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rte(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtp(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtn(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtz(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rte(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtp(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtn(half2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtz(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rte(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtp(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtn(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtz(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rte(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtp(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtn(float2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtz(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rte(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtp(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_rtn(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtz(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rte(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtp(double2); __attribute__((overloadable)) ulong2 __spirv_ConvertFToU_Rulong2_sat_rtn(double2); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtz(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rte(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtp(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtn(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtz(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rte(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtp(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtn(half3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtz(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rte(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtp(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtn(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtz(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rte(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtp(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtn(float3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtz(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rte(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtp(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_rtn(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtz(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rte(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtp(double3); __attribute__((overloadable)) ulong3 __spirv_ConvertFToU_Rulong3_sat_rtn(double3); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtz(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rte(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtp(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtn(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtz(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rte(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtp(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtn(half4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtz(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rte(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtp(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtn(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtz(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rte(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtp(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtn(float4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtz(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rte(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtp(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_rtn(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtz(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rte(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtp(double4); __attribute__((overloadable)) ulong4 __spirv_ConvertFToU_Rulong4_sat_rtn(double4); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtz(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rte(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtp(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtn(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtz(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rte(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtp(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtn(half8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtz(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rte(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtp(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtn(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtz(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rte(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtp(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtn(float8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtz(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rte(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtp(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_rtn(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtz(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rte(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtp(double8); __attribute__((overloadable)) ulong8 __spirv_ConvertFToU_Rulong8_sat_rtn(double8); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtz(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rte(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtp(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtn(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtz(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rte(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtp(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtn(half16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtz(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rte(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtp(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtn(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtz(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rte(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtp(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtn(float16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtz(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rte(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtp(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_rtn(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtz(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rte(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtp(double16); __attribute__((overloadable)) ulong16 __spirv_ConvertFToU_Rulong16_sat_rtn(double16); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtz(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rte(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtp(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtn(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtz(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rte(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtp(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtn(half); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtz(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rte(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtp(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtn(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtz(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rte(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtp(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtn(float); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtz(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rte(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtp(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_rtn(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtz(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rte(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtp(double); __attribute__((overloadable)) char __spirv_ConvertFToS_Rchar_sat_rtn(double); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtz(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rte(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtp(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtn(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtz(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rte(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtp(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtn(half2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtz(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rte(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtp(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtn(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtz(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rte(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtp(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtn(float2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtz(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rte(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtp(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_rtn(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtz(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rte(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtp(double2); __attribute__((overloadable)) char2 __spirv_ConvertFToS_Rchar2_sat_rtn(double2); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtz(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rte(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtp(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtn(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtz(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rte(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtp(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtn(half3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtz(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rte(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtp(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtn(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtz(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rte(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtp(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtn(float3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtz(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rte(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtp(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_rtn(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtz(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rte(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtp(double3); __attribute__((overloadable)) char3 __spirv_ConvertFToS_Rchar3_sat_rtn(double3); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtz(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rte(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtp(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtn(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtz(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rte(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtp(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtn(half4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtz(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rte(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtp(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtn(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtz(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rte(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtp(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtn(float4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtz(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rte(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtp(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_rtn(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtz(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rte(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtp(double4); __attribute__((overloadable)) char4 __spirv_ConvertFToS_Rchar4_sat_rtn(double4); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtz(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rte(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtp(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtn(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtz(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rte(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtp(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtn(half8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtz(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rte(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtp(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtn(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtz(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rte(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtp(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtn(float8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtz(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rte(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtp(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_rtn(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtz(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rte(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtp(double8); __attribute__((overloadable)) char8 __spirv_ConvertFToS_Rchar8_sat_rtn(double8); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtz(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rte(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtp(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtn(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtz(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rte(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtp(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtn(half16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtz(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rte(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtp(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtn(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtz(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rte(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtp(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtn(float16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtz(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rte(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtp(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_rtn(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtz(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rte(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtp(double16); __attribute__((overloadable)) char16 __spirv_ConvertFToS_Rchar16_sat_rtn(double16); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtz(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rte(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtp(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtn(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtz(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rte(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtp(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtn(half); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtz(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rte(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtp(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtn(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtz(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rte(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtp(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtn(float); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtz(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rte(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtp(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_rtn(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtz(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rte(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtp(double); __attribute__((overloadable)) short __spirv_ConvertFToS_Rshort_sat_rtn(double); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtz(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rte(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtp(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtn(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtz(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rte(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtp(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtn(half2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtz(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rte(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtp(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtn(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtz(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rte(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtp(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtn(float2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtz(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rte(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtp(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_rtn(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtz(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rte(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtp(double2); __attribute__((overloadable)) short2 __spirv_ConvertFToS_Rshort2_sat_rtn(double2); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtz(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rte(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtp(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtn(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtz(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rte(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtp(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtn(half3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtz(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rte(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtp(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtn(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtz(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rte(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtp(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtn(float3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtz(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rte(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtp(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_rtn(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtz(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rte(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtp(double3); __attribute__((overloadable)) short3 __spirv_ConvertFToS_Rshort3_sat_rtn(double3); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtz(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rte(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtp(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtn(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtz(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rte(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtp(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtn(half4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtz(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rte(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtp(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtn(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtz(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rte(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtp(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtn(float4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtz(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rte(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtp(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_rtn(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtz(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rte(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtp(double4); __attribute__((overloadable)) short4 __spirv_ConvertFToS_Rshort4_sat_rtn(double4); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtz(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rte(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtp(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtn(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtz(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rte(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtp(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtn(half8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtz(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rte(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtp(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtn(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtz(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rte(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtp(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtn(float8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtz(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rte(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtp(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_rtn(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtz(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rte(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtp(double8); __attribute__((overloadable)) short8 __spirv_ConvertFToS_Rshort8_sat_rtn(double8); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtz(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rte(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtp(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtn(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtz(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rte(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtp(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtn(half16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtz(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rte(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtp(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtn(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtz(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rte(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtp(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtn(float16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtz(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rte(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtp(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_rtn(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtz(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rte(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtp(double16); __attribute__((overloadable)) short16 __spirv_ConvertFToS_Rshort16_sat_rtn(double16); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtz(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rte(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtp(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtn(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtz(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rte(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtp(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtn(half); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtz(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rte(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtp(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtn(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtz(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rte(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtp(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtn(float); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtz(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rte(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtp(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_rtn(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtz(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rte(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtp(double); __attribute__((overloadable)) int __spirv_ConvertFToS_Rint_sat_rtn(double); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtz(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rte(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtp(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtn(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtz(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rte(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtp(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtn(half2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtz(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rte(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtp(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtn(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtz(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rte(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtp(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtn(float2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtz(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rte(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtp(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_rtn(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtz(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rte(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtp(double2); __attribute__((overloadable)) int2 __spirv_ConvertFToS_Rint2_sat_rtn(double2); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtz(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rte(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtp(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtn(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtz(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rte(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtp(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtn(half3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtz(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rte(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtp(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtn(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtz(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rte(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtp(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtn(float3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtz(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rte(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtp(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_rtn(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtz(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rte(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtp(double3); __attribute__((overloadable)) int3 __spirv_ConvertFToS_Rint3_sat_rtn(double3); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtz(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rte(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtp(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtn(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtz(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rte(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtp(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtn(half4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtz(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rte(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtp(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtn(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtz(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rte(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtp(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtn(float4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtz(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rte(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtp(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_rtn(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtz(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rte(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtp(double4); __attribute__((overloadable)) int4 __spirv_ConvertFToS_Rint4_sat_rtn(double4); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtz(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rte(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtp(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtn(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtz(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rte(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtp(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtn(half8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtz(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rte(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtp(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtn(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtz(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rte(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtp(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtn(float8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtz(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rte(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtp(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_rtn(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtz(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rte(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtp(double8); __attribute__((overloadable)) int8 __spirv_ConvertFToS_Rint8_sat_rtn(double8); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtz(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rte(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtp(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtn(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtz(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rte(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtp(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtn(half16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtz(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rte(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtp(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtn(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtz(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rte(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtp(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtn(float16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtz(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rte(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtp(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_rtn(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtz(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rte(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtp(double16); __attribute__((overloadable)) int16 __spirv_ConvertFToS_Rint16_sat_rtn(double16); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtz(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rte(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtp(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtn(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtz(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rte(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtp(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtn(half); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtz(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rte(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtp(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtn(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtz(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rte(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtp(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtn(float); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtz(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rte(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtp(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_rtn(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtz(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rte(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtp(double); __attribute__((overloadable)) long __spirv_ConvertFToS_Rlong_sat_rtn(double); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtz(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rte(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtp(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtn(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtz(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rte(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtp(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtn(half2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtz(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rte(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtp(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtn(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtz(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rte(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtp(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtn(float2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtz(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rte(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtp(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_rtn(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtz(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rte(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtp(double2); __attribute__((overloadable)) long2 __spirv_ConvertFToS_Rlong2_sat_rtn(double2); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtz(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rte(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtp(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtn(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtz(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rte(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtp(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtn(half3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtz(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rte(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtp(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtn(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtz(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rte(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtp(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtn(float3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtz(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rte(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtp(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_rtn(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtz(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rte(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtp(double3); __attribute__((overloadable)) long3 __spirv_ConvertFToS_Rlong3_sat_rtn(double3); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtz(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rte(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtp(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtn(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtz(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rte(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtp(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtn(half4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtz(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rte(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtp(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtn(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtz(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rte(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtp(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtn(float4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtz(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rte(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtp(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_rtn(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtz(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rte(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtp(double4); __attribute__((overloadable)) long4 __spirv_ConvertFToS_Rlong4_sat_rtn(double4); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtz(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rte(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtp(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtn(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtz(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rte(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtp(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtn(half8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtz(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rte(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtp(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtn(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtz(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rte(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtp(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtn(float8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtz(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rte(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtp(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_rtn(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtz(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rte(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtp(double8); __attribute__((overloadable)) long8 __spirv_ConvertFToS_Rlong8_sat_rtn(double8); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtz(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rte(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtp(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtn(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtz(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rte(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtp(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtn(half16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtz(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rte(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtp(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtn(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtz(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rte(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtp(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtn(float16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtz(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rte(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtp(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_rtn(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtz(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rte(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtp(double16); __attribute__((overloadable)) long16 __spirv_ConvertFToS_Rlong16_sat_rtn(double16); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf(uchar); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtz(uchar); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rte(uchar); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtp(uchar); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtn(uchar); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf(ushort); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtz(ushort); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rte(ushort); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtp(ushort); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtn(ushort); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf(uint); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtz(uint); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rte(uint); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtp(uint); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtn(uint); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf(ulong); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtz(ulong); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rte(ulong); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtp(ulong); __attribute__((overloadable)) half __spirv_ConvertUToF_Rhalf_rtn(ulong); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2(uchar2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtz(uchar2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rte(uchar2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtp(uchar2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtn(uchar2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2(ushort2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtz(ushort2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rte(ushort2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtp(ushort2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtn(ushort2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2(uint2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtz(uint2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rte(uint2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtp(uint2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtn(uint2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2(ulong2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtz(ulong2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rte(ulong2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtp(ulong2); __attribute__((overloadable)) half2 __spirv_ConvertUToF_Rhalf2_rtn(ulong2); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3(uchar3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtz(uchar3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rte(uchar3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtp(uchar3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtn(uchar3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3(ushort3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtz(ushort3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rte(ushort3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtp(ushort3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtn(ushort3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3(uint3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtz(uint3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rte(uint3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtp(uint3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtn(uint3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3(ulong3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtz(ulong3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rte(ulong3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtp(ulong3); __attribute__((overloadable)) half3 __spirv_ConvertUToF_Rhalf3_rtn(ulong3); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4(uchar4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtz(uchar4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rte(uchar4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtp(uchar4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtn(uchar4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4(ushort4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtz(ushort4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rte(ushort4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtp(ushort4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtn(ushort4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4(uint4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtz(uint4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rte(uint4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtp(uint4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtn(uint4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4(ulong4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtz(ulong4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rte(ulong4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtp(ulong4); __attribute__((overloadable)) half4 __spirv_ConvertUToF_Rhalf4_rtn(ulong4); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8(uchar8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtz(uchar8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rte(uchar8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtp(uchar8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtn(uchar8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8(ushort8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtz(ushort8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rte(ushort8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtp(ushort8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtn(ushort8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8(uint8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtz(uint8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rte(uint8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtp(uint8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtn(uint8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8(ulong8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtz(ulong8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rte(ulong8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtp(ulong8); __attribute__((overloadable)) half8 __spirv_ConvertUToF_Rhalf8_rtn(ulong8); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16(uchar16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtz(uchar16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rte(uchar16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtp(uchar16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtn(uchar16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16(ushort16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtz(ushort16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rte(ushort16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtp(ushort16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtn(ushort16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16(uint16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtz(uint16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rte(uint16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtp(uint16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtn(uint16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16(ulong16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtz(ulong16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rte(ulong16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtp(ulong16); __attribute__((overloadable)) half16 __spirv_ConvertUToF_Rhalf16_rtn(ulong16); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat(uchar); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtz(uchar); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rte(uchar); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtp(uchar); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtn(uchar); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat(ushort); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtz(ushort); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rte(ushort); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtp(ushort); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtn(ushort); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat(uint); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtz(uint); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rte(uint); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtp(uint); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtn(uint); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat(ulong); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtz(ulong); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rte(ulong); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtp(ulong); __attribute__((overloadable)) float __spirv_ConvertUToF_Rfloat_rtn(ulong); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2(uchar2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtz(uchar2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rte(uchar2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtp(uchar2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtn(uchar2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2(ushort2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtz(ushort2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rte(ushort2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtp(ushort2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtn(ushort2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2(uint2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtz(uint2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rte(uint2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtp(uint2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtn(uint2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2(ulong2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtz(ulong2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rte(ulong2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtp(ulong2); __attribute__((overloadable)) float2 __spirv_ConvertUToF_Rfloat2_rtn(ulong2); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3(uchar3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtz(uchar3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rte(uchar3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtp(uchar3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtn(uchar3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3(ushort3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtz(ushort3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rte(ushort3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtp(ushort3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtn(ushort3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3(uint3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtz(uint3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rte(uint3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtp(uint3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtn(uint3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3(ulong3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtz(ulong3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rte(ulong3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtp(ulong3); __attribute__((overloadable)) float3 __spirv_ConvertUToF_Rfloat3_rtn(ulong3); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4(uchar4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtz(uchar4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rte(uchar4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtp(uchar4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtn(uchar4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4(ushort4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtz(ushort4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rte(ushort4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtp(ushort4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtn(ushort4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4(uint4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtz(uint4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rte(uint4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtp(uint4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtn(uint4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4(ulong4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtz(ulong4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rte(ulong4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtp(ulong4); __attribute__((overloadable)) float4 __spirv_ConvertUToF_Rfloat4_rtn(ulong4); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8(uchar8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtz(uchar8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rte(uchar8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtp(uchar8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtn(uchar8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8(ushort8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtz(ushort8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rte(ushort8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtp(ushort8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtn(ushort8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8(uint8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtz(uint8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rte(uint8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtp(uint8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtn(uint8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8(ulong8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtz(ulong8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rte(ulong8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtp(ulong8); __attribute__((overloadable)) float8 __spirv_ConvertUToF_Rfloat8_rtn(ulong8); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16(uchar16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtz(uchar16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rte(uchar16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtp(uchar16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtn(uchar16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16(ushort16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtz(ushort16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rte(ushort16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtp(ushort16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtn(ushort16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16(uint16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtz(uint16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rte(uint16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtp(uint16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtn(uint16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16(ulong16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtz(ulong16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rte(ulong16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtp(ulong16); __attribute__((overloadable)) float16 __spirv_ConvertUToF_Rfloat16_rtn(ulong16); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble(uchar); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtz(uchar); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rte(uchar); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtp(uchar); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtn(uchar); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble(ushort); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtz(ushort); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rte(ushort); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtp(ushort); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtn(ushort); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble(uint); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtz(uint); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rte(uint); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtp(uint); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtn(uint); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble(ulong); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtz(ulong); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rte(ulong); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtp(ulong); __attribute__((overloadable)) double __spirv_ConvertUToF_Rdouble_rtn(ulong); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2(uchar2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtz(uchar2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rte(uchar2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtp(uchar2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtn(uchar2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2(ushort2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtz(ushort2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rte(ushort2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtp(ushort2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtn(ushort2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2(uint2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtz(uint2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rte(uint2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtp(uint2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtn(uint2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2(ulong2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtz(ulong2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rte(ulong2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtp(ulong2); __attribute__((overloadable)) double2 __spirv_ConvertUToF_Rdouble2_rtn(ulong2); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3(uchar3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtz(uchar3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rte(uchar3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtp(uchar3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtn(uchar3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3(ushort3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtz(ushort3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rte(ushort3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtp(ushort3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtn(ushort3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3(uint3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtz(uint3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rte(uint3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtp(uint3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtn(uint3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3(ulong3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtz(ulong3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rte(ulong3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtp(ulong3); __attribute__((overloadable)) double3 __spirv_ConvertUToF_Rdouble3_rtn(ulong3); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4(uchar4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtz(uchar4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rte(uchar4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtp(uchar4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtn(uchar4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4(ushort4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtz(ushort4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rte(ushort4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtp(ushort4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtn(ushort4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4(uint4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtz(uint4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rte(uint4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtp(uint4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtn(uint4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4(ulong4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtz(ulong4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rte(ulong4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtp(ulong4); __attribute__((overloadable)) double4 __spirv_ConvertUToF_Rdouble4_rtn(ulong4); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8(uchar8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtz(uchar8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rte(uchar8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtp(uchar8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtn(uchar8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8(ushort8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtz(ushort8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rte(ushort8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtp(ushort8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtn(ushort8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8(uint8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtz(uint8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rte(uint8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtp(uint8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtn(uint8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8(ulong8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtz(ulong8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rte(ulong8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtp(ulong8); __attribute__((overloadable)) double8 __spirv_ConvertUToF_Rdouble8_rtn(ulong8); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16(uchar16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtz(uchar16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rte(uchar16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtp(uchar16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtn(uchar16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16(ushort16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtz(ushort16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rte(ushort16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtp(ushort16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtn(ushort16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16(uint16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtz(uint16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rte(uint16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtp(uint16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtn(uint16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16(ulong16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtz(ulong16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rte(ulong16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtp(ulong16); __attribute__((overloadable)) double16 __spirv_ConvertUToF_Rdouble16_rtn(ulong16); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf(char); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtz(char); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rte(char); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtp(char); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtn(char); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf(short); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtz(short); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rte(short); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtp(short); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtn(short); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf(int); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtz(int); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rte(int); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtp(int); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtn(int); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf(long); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtz(long); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rte(long); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtp(long); __attribute__((overloadable)) half __spirv_ConvertSToF_Rhalf_rtn(long); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2(char2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtz(char2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rte(char2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtp(char2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtn(char2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2(short2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtz(short2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rte(short2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtp(short2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtn(short2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2(int2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtz(int2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rte(int2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtp(int2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtn(int2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2(long2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtz(long2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rte(long2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtp(long2); __attribute__((overloadable)) half2 __spirv_ConvertSToF_Rhalf2_rtn(long2); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3(char3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtz(char3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rte(char3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtp(char3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtn(char3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3(short3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtz(short3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rte(short3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtp(short3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtn(short3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3(int3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtz(int3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rte(int3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtp(int3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtn(int3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3(long3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtz(long3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rte(long3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtp(long3); __attribute__((overloadable)) half3 __spirv_ConvertSToF_Rhalf3_rtn(long3); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4(char4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtz(char4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rte(char4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtp(char4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtn(char4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4(short4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtz(short4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rte(short4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtp(short4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtn(short4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4(int4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtz(int4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rte(int4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtp(int4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtn(int4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4(long4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtz(long4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rte(long4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtp(long4); __attribute__((overloadable)) half4 __spirv_ConvertSToF_Rhalf4_rtn(long4); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8(char8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtz(char8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rte(char8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtp(char8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtn(char8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8(short8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtz(short8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rte(short8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtp(short8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtn(short8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8(int8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtz(int8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rte(int8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtp(int8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtn(int8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8(long8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtz(long8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rte(long8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtp(long8); __attribute__((overloadable)) half8 __spirv_ConvertSToF_Rhalf8_rtn(long8); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16(char16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtz(char16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rte(char16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtp(char16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtn(char16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16(short16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtz(short16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rte(short16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtp(short16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtn(short16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16(int16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtz(int16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rte(int16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtp(int16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtn(int16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16(long16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtz(long16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rte(long16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtp(long16); __attribute__((overloadable)) half16 __spirv_ConvertSToF_Rhalf16_rtn(long16); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat(char); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtz(char); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rte(char); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtp(char); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtn(char); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat(short); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtz(short); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rte(short); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtp(short); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtn(short); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat(int); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtz(int); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rte(int); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtp(int); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtn(int); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat(long); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtz(long); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rte(long); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtp(long); __attribute__((overloadable)) float __spirv_ConvertSToF_Rfloat_rtn(long); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2(char2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtz(char2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rte(char2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtp(char2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtn(char2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2(short2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtz(short2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rte(short2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtp(short2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtn(short2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2(int2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtz(int2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rte(int2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtp(int2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtn(int2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2(long2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtz(long2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rte(long2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtp(long2); __attribute__((overloadable)) float2 __spirv_ConvertSToF_Rfloat2_rtn(long2); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3(char3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtz(char3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rte(char3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtp(char3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtn(char3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3(short3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtz(short3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rte(short3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtp(short3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtn(short3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3(int3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtz(int3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rte(int3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtp(int3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtn(int3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3(long3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtz(long3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rte(long3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtp(long3); __attribute__((overloadable)) float3 __spirv_ConvertSToF_Rfloat3_rtn(long3); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4(char4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtz(char4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rte(char4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtp(char4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtn(char4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4(short4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtz(short4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rte(short4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtp(short4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtn(short4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4(int4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtz(int4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rte(int4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtp(int4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtn(int4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4(long4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtz(long4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rte(long4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtp(long4); __attribute__((overloadable)) float4 __spirv_ConvertSToF_Rfloat4_rtn(long4); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8(char8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtz(char8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rte(char8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtp(char8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtn(char8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8(short8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtz(short8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rte(short8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtp(short8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtn(short8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8(int8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtz(int8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rte(int8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtp(int8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtn(int8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8(long8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtz(long8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rte(long8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtp(long8); __attribute__((overloadable)) float8 __spirv_ConvertSToF_Rfloat8_rtn(long8); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16(char16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtz(char16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rte(char16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtp(char16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtn(char16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16(short16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtz(short16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rte(short16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtp(short16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtn(short16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16(int16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtz(int16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rte(int16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtp(int16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtn(int16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16(long16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtz(long16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rte(long16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtp(long16); __attribute__((overloadable)) float16 __spirv_ConvertSToF_Rfloat16_rtn(long16); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble(char); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtz(char); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rte(char); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtp(char); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtn(char); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble(short); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtz(short); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rte(short); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtp(short); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtn(short); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble(int); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtz(int); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rte(int); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtp(int); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtn(int); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble(long); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtz(long); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rte(long); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtp(long); __attribute__((overloadable)) double __spirv_ConvertSToF_Rdouble_rtn(long); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2(char2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtz(char2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rte(char2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtp(char2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtn(char2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2(short2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtz(short2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rte(short2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtp(short2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtn(short2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2(int2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtz(int2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rte(int2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtp(int2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtn(int2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2(long2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtz(long2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rte(long2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtp(long2); __attribute__((overloadable)) double2 __spirv_ConvertSToF_Rdouble2_rtn(long2); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3(char3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtz(char3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rte(char3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtp(char3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtn(char3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3(short3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtz(short3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rte(short3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtp(short3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtn(short3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3(int3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtz(int3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rte(int3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtp(int3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtn(int3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3(long3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtz(long3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rte(long3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtp(long3); __attribute__((overloadable)) double3 __spirv_ConvertSToF_Rdouble3_rtn(long3); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4(char4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtz(char4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rte(char4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtp(char4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtn(char4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4(short4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtz(short4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rte(short4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtp(short4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtn(short4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4(int4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtz(int4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rte(int4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtp(int4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtn(int4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4(long4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtz(long4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rte(long4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtp(long4); __attribute__((overloadable)) double4 __spirv_ConvertSToF_Rdouble4_rtn(long4); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8(char8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtz(char8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rte(char8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtp(char8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtn(char8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8(short8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtz(short8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rte(short8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtp(short8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtn(short8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8(int8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtz(int8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rte(int8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtp(int8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtn(int8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8(long8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtz(long8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rte(long8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtp(long8); __attribute__((overloadable)) double8 __spirv_ConvertSToF_Rdouble8_rtn(long8); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16(char16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtz(char16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rte(char16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtp(char16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtn(char16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16(short16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtz(short16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rte(short16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtp(short16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtn(short16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16(int16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtz(int16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rte(int16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtp(int16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtn(int16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16(long16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtz(long16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rte(long16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtp(long16); __attribute__((overloadable)) double16 __spirv_ConvertSToF_Rdouble16_rtn(long16); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtz(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rte(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtp(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtn(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtz(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rte(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtp(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtn(uchar); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtz(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rte(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtp(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtn(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtz(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rte(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtp(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtn(ushort); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtz(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rte(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtp(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtn(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtz(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rte(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtp(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtn(uint); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtz(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rte(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtp(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_rtn(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtz(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rte(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtp(ulong); __attribute__((overloadable)) uchar __spirv_UConvert_Ruchar_sat_rtn(ulong); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtz(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rte(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtp(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtn(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtz(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rte(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtp(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtn(uchar2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtz(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rte(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtp(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtn(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtz(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rte(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtp(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtn(ushort2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtz(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rte(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtp(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtn(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtz(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rte(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtp(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtn(uint2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtz(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rte(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtp(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_rtn(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtz(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rte(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtp(ulong2); __attribute__((overloadable)) uchar2 __spirv_UConvert_Ruchar2_sat_rtn(ulong2); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtz(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rte(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtp(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtn(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtz(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rte(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtp(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtn(uchar3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtz(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rte(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtp(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtn(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtz(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rte(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtp(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtn(ushort3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtz(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rte(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtp(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtn(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtz(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rte(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtp(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtn(uint3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtz(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rte(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtp(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_rtn(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtz(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rte(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtp(ulong3); __attribute__((overloadable)) uchar3 __spirv_UConvert_Ruchar3_sat_rtn(ulong3); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtz(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rte(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtp(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtn(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtz(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rte(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtp(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtn(uchar4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtz(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rte(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtp(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtn(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtz(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rte(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtp(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtn(ushort4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtz(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rte(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtp(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtn(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtz(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rte(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtp(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtn(uint4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtz(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rte(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtp(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_rtn(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtz(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rte(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtp(ulong4); __attribute__((overloadable)) uchar4 __spirv_UConvert_Ruchar4_sat_rtn(ulong4); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtz(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rte(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtp(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtn(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtz(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rte(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtp(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtn(uchar8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtz(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rte(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtp(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtn(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtz(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rte(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtp(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtn(ushort8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtz(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rte(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtp(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtn(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtz(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rte(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtp(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtn(uint8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtz(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rte(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtp(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_rtn(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtz(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rte(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtp(ulong8); __attribute__((overloadable)) uchar8 __spirv_UConvert_Ruchar8_sat_rtn(ulong8); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtz(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rte(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtp(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtn(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtz(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rte(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtp(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtn(uchar16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtz(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rte(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtp(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtn(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtz(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rte(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtp(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtn(ushort16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtz(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rte(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtp(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtn(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtz(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rte(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtp(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtn(uint16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtz(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rte(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtp(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_rtn(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtz(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rte(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtp(ulong16); __attribute__((overloadable)) uchar16 __spirv_UConvert_Ruchar16_sat_rtn(ulong16); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtz(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rte(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtp(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtn(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtz(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rte(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtp(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtn(uchar); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtz(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rte(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtp(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtn(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtz(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rte(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtp(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtn(ushort); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtz(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rte(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtp(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtn(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtz(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rte(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtp(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtn(uint); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtz(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rte(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtp(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_rtn(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtz(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rte(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtp(ulong); __attribute__((overloadable)) ushort __spirv_UConvert_Rushort_sat_rtn(ulong); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtz(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rte(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtp(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtn(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtz(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rte(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtp(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtn(uchar2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtz(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rte(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtp(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtn(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtz(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rte(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtp(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtn(ushort2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtz(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rte(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtp(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtn(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtz(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rte(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtp(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtn(uint2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtz(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rte(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtp(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_rtn(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtz(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rte(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtp(ulong2); __attribute__((overloadable)) ushort2 __spirv_UConvert_Rushort2_sat_rtn(ulong2); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtz(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rte(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtp(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtn(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtz(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rte(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtp(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtn(uchar3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtz(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rte(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtp(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtn(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtz(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rte(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtp(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtn(ushort3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtz(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rte(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtp(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtn(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtz(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rte(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtp(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtn(uint3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtz(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rte(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtp(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_rtn(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtz(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rte(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtp(ulong3); __attribute__((overloadable)) ushort3 __spirv_UConvert_Rushort3_sat_rtn(ulong3); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtz(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rte(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtp(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtn(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtz(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rte(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtp(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtn(uchar4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtz(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rte(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtp(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtn(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtz(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rte(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtp(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtn(ushort4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtz(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rte(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtp(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtn(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtz(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rte(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtp(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtn(uint4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtz(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rte(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtp(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_rtn(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtz(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rte(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtp(ulong4); __attribute__((overloadable)) ushort4 __spirv_UConvert_Rushort4_sat_rtn(ulong4); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtz(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rte(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtp(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtn(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtz(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rte(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtp(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtn(uchar8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtz(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rte(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtp(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtn(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtz(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rte(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtp(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtn(ushort8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtz(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rte(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtp(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtn(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtz(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rte(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtp(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtn(uint8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtz(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rte(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtp(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_rtn(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtz(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rte(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtp(ulong8); __attribute__((overloadable)) ushort8 __spirv_UConvert_Rushort8_sat_rtn(ulong8); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtz(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rte(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtp(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtn(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtz(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rte(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtp(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtn(uchar16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtz(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rte(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtp(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtn(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtz(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rte(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtp(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtn(ushort16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtz(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rte(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtp(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtn(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtz(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rte(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtp(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtn(uint16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtz(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rte(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtp(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_rtn(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtz(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rte(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtp(ulong16); __attribute__((overloadable)) ushort16 __spirv_UConvert_Rushort16_sat_rtn(ulong16); __attribute__((overloadable)) uint __spirv_UConvert_Ruint(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtz(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rte(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtp(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtn(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtz(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rte(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtp(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtn(uchar); __attribute__((overloadable)) uint __spirv_UConvert_Ruint(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtz(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rte(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtp(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtn(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtz(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rte(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtp(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtn(ushort); __attribute__((overloadable)) uint __spirv_UConvert_Ruint(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtz(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rte(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtp(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtn(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtz(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rte(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtp(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtn(uint); __attribute__((overloadable)) uint __spirv_UConvert_Ruint(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtz(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rte(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtp(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_rtn(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtz(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rte(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtp(ulong); __attribute__((overloadable)) uint __spirv_UConvert_Ruint_sat_rtn(ulong); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtz(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rte(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtp(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtn(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtz(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rte(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtp(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtn(uchar2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtz(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rte(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtp(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtn(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtz(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rte(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtp(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtn(ushort2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtz(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rte(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtp(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtn(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtz(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rte(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtp(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtn(uint2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtz(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rte(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtp(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_rtn(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtz(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rte(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtp(ulong2); __attribute__((overloadable)) uint2 __spirv_UConvert_Ruint2_sat_rtn(ulong2); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtz(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rte(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtp(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtn(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtz(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rte(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtp(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtn(uchar3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtz(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rte(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtp(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtn(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtz(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rte(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtp(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtn(ushort3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtz(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rte(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtp(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtn(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtz(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rte(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtp(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtn(uint3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtz(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rte(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtp(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_rtn(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtz(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rte(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtp(ulong3); __attribute__((overloadable)) uint3 __spirv_UConvert_Ruint3_sat_rtn(ulong3); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtz(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rte(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtp(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtn(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtz(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rte(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtp(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtn(uchar4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtz(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rte(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtp(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtn(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtz(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rte(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtp(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtn(ushort4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtz(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rte(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtp(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtn(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtz(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rte(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtp(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtn(uint4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtz(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rte(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtp(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_rtn(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtz(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rte(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtp(ulong4); __attribute__((overloadable)) uint4 __spirv_UConvert_Ruint4_sat_rtn(ulong4); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtz(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rte(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtp(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtn(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtz(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rte(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtp(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtn(uchar8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtz(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rte(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtp(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtn(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtz(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rte(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtp(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtn(ushort8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtz(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rte(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtp(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtn(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtz(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rte(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtp(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtn(uint8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtz(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rte(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtp(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_rtn(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtz(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rte(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtp(ulong8); __attribute__((overloadable)) uint8 __spirv_UConvert_Ruint8_sat_rtn(ulong8); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtz(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rte(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtp(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtn(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtz(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rte(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtp(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtn(uchar16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtz(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rte(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtp(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtn(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtz(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rte(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtp(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtn(ushort16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtz(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rte(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtp(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtn(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtz(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rte(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtp(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtn(uint16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtz(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rte(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtp(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_rtn(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtz(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rte(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtp(ulong16); __attribute__((overloadable)) uint16 __spirv_UConvert_Ruint16_sat_rtn(ulong16); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtz(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rte(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtp(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtn(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtz(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rte(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtp(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtn(uchar); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtz(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rte(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtp(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtn(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtz(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rte(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtp(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtn(ushort); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtz(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rte(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtp(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtn(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtz(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rte(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtp(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtn(uint); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtz(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rte(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtp(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_rtn(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtz(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rte(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtp(ulong); __attribute__((overloadable)) ulong __spirv_UConvert_Rulong_sat_rtn(ulong); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtz(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rte(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtp(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtn(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtz(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rte(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtp(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtn(uchar2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtz(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rte(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtp(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtn(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtz(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rte(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtp(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtn(ushort2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtz(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rte(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtp(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtn(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtz(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rte(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtp(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtn(uint2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtz(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rte(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtp(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_rtn(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtz(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rte(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtp(ulong2); __attribute__((overloadable)) ulong2 __spirv_UConvert_Rulong2_sat_rtn(ulong2); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtz(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rte(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtp(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtn(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtz(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rte(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtp(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtn(uchar3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtz(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rte(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtp(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtn(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtz(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rte(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtp(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtn(ushort3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtz(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rte(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtp(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtn(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtz(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rte(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtp(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtn(uint3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtz(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rte(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtp(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_rtn(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtz(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rte(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtp(ulong3); __attribute__((overloadable)) ulong3 __spirv_UConvert_Rulong3_sat_rtn(ulong3); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtz(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rte(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtp(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtn(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtz(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rte(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtp(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtn(uchar4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtz(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rte(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtp(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtn(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtz(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rte(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtp(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtn(ushort4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtz(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rte(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtp(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtn(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtz(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rte(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtp(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtn(uint4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtz(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rte(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtp(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_rtn(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtz(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rte(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtp(ulong4); __attribute__((overloadable)) ulong4 __spirv_UConvert_Rulong4_sat_rtn(ulong4); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtz(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rte(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtp(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtn(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtz(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rte(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtp(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtn(uchar8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtz(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rte(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtp(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtn(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtz(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rte(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtp(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtn(ushort8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtz(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rte(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtp(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtn(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtz(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rte(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtp(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtn(uint8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtz(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rte(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtp(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_rtn(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtz(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rte(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtp(ulong8); __attribute__((overloadable)) ulong8 __spirv_UConvert_Rulong8_sat_rtn(ulong8); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtz(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rte(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtp(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtn(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtz(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rte(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtp(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtn(uchar16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtz(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rte(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtp(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtn(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtz(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rte(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtp(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtn(ushort16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtz(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rte(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtp(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtn(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtz(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rte(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtp(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtn(uint16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtz(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rte(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtp(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_rtn(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtz(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rte(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtp(ulong16); __attribute__((overloadable)) ulong16 __spirv_UConvert_Rulong16_sat_rtn(ulong16); __attribute__((overloadable)) char __spirv_SConvert_Rchar(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtz(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rte(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtp(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtn(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtz(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rte(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtp(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtn(char); __attribute__((overloadable)) char __spirv_SConvert_Rchar(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtz(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rte(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtp(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtn(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtz(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rte(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtp(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtn(short); __attribute__((overloadable)) char __spirv_SConvert_Rchar(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtz(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rte(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtp(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtn(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtz(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rte(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtp(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtn(int); __attribute__((overloadable)) char __spirv_SConvert_Rchar(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtz(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rte(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtp(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_rtn(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtz(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rte(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtp(long); __attribute__((overloadable)) char __spirv_SConvert_Rchar_sat_rtn(long); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtz(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rte(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtp(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtn(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtz(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rte(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtp(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtn(char2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtz(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rte(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtp(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtn(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtz(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rte(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtp(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtn(short2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtz(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rte(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtp(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtn(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtz(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rte(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtp(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtn(int2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtz(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rte(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtp(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_rtn(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtz(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rte(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtp(long2); __attribute__((overloadable)) char2 __spirv_SConvert_Rchar2_sat_rtn(long2); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtz(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rte(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtp(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtn(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtz(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rte(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtp(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtn(char3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtz(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rte(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtp(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtn(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtz(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rte(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtp(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtn(short3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtz(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rte(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtp(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtn(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtz(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rte(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtp(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtn(int3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtz(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rte(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtp(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_rtn(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtz(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rte(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtp(long3); __attribute__((overloadable)) char3 __spirv_SConvert_Rchar3_sat_rtn(long3); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtz(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rte(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtp(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtn(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtz(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rte(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtp(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtn(char4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtz(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rte(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtp(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtn(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtz(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rte(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtp(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtn(short4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtz(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rte(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtp(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtn(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtz(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rte(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtp(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtn(int4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtz(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rte(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtp(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_rtn(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtz(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rte(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtp(long4); __attribute__((overloadable)) char4 __spirv_SConvert_Rchar4_sat_rtn(long4); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtz(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rte(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtp(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtn(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtz(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rte(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtp(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtn(char8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtz(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rte(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtp(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtn(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtz(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rte(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtp(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtn(short8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtz(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rte(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtp(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtn(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtz(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rte(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtp(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtn(int8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtz(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rte(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtp(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_rtn(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtz(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rte(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtp(long8); __attribute__((overloadable)) char8 __spirv_SConvert_Rchar8_sat_rtn(long8); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtz(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rte(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtp(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtn(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtz(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rte(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtp(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtn(char16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtz(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rte(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtp(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtn(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtz(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rte(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtp(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtn(short16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtz(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rte(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtp(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtn(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtz(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rte(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtp(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtn(int16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtz(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rte(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtp(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_rtn(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtz(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rte(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtp(long16); __attribute__((overloadable)) char16 __spirv_SConvert_Rchar16_sat_rtn(long16); __attribute__((overloadable)) short __spirv_SConvert_Rshort(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtz(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rte(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtp(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtn(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtz(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rte(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtp(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtn(char); __attribute__((overloadable)) short __spirv_SConvert_Rshort(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtz(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rte(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtp(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtn(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtz(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rte(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtp(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtn(short); __attribute__((overloadable)) short __spirv_SConvert_Rshort(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtz(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rte(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtp(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtn(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtz(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rte(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtp(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtn(int); __attribute__((overloadable)) short __spirv_SConvert_Rshort(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtz(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rte(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtp(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_rtn(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtz(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rte(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtp(long); __attribute__((overloadable)) short __spirv_SConvert_Rshort_sat_rtn(long); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtz(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rte(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtp(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtn(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtz(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rte(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtp(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtn(char2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtz(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rte(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtp(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtn(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtz(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rte(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtp(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtn(short2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtz(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rte(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtp(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtn(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtz(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rte(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtp(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtn(int2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtz(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rte(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtp(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_rtn(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtz(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rte(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtp(long2); __attribute__((overloadable)) short2 __spirv_SConvert_Rshort2_sat_rtn(long2); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtz(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rte(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtp(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtn(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtz(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rte(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtp(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtn(char3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtz(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rte(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtp(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtn(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtz(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rte(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtp(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtn(short3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtz(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rte(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtp(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtn(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtz(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rte(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtp(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtn(int3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtz(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rte(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtp(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_rtn(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtz(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rte(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtp(long3); __attribute__((overloadable)) short3 __spirv_SConvert_Rshort3_sat_rtn(long3); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtz(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rte(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtp(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtn(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtz(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rte(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtp(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtn(char4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtz(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rte(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtp(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtn(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtz(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rte(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtp(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtn(short4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtz(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rte(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtp(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtn(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtz(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rte(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtp(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtn(int4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtz(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rte(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtp(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_rtn(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtz(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rte(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtp(long4); __attribute__((overloadable)) short4 __spirv_SConvert_Rshort4_sat_rtn(long4); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtz(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rte(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtp(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtn(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtz(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rte(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtp(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtn(char8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtz(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rte(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtp(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtn(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtz(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rte(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtp(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtn(short8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtz(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rte(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtp(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtn(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtz(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rte(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtp(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtn(int8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtz(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rte(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtp(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_rtn(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtz(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rte(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtp(long8); __attribute__((overloadable)) short8 __spirv_SConvert_Rshort8_sat_rtn(long8); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtz(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rte(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtp(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtn(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtz(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rte(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtp(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtn(char16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtz(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rte(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtp(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtn(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtz(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rte(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtp(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtn(short16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtz(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rte(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtp(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtn(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtz(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rte(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtp(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtn(int16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtz(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rte(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtp(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_rtn(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtz(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rte(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtp(long16); __attribute__((overloadable)) short16 __spirv_SConvert_Rshort16_sat_rtn(long16); __attribute__((overloadable)) int __spirv_SConvert_Rint(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtz(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_rte(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtp(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtn(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtz(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rte(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtp(char); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtn(char); __attribute__((overloadable)) int __spirv_SConvert_Rint(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtz(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_rte(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtp(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtn(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtz(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rte(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtp(short); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtn(short); __attribute__((overloadable)) int __spirv_SConvert_Rint(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtz(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_rte(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtp(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtn(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtz(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rte(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtp(int); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtn(int); __attribute__((overloadable)) int __spirv_SConvert_Rint(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtz(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_rte(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtp(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_rtn(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtz(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rte(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtp(long); __attribute__((overloadable)) int __spirv_SConvert_Rint_sat_rtn(long); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtz(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rte(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtp(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtn(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtz(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rte(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtp(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtn(char2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtz(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rte(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtp(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtn(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtz(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rte(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtp(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtn(short2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtz(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rte(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtp(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtn(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtz(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rte(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtp(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtn(int2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtz(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rte(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtp(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_rtn(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtz(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rte(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtp(long2); __attribute__((overloadable)) int2 __spirv_SConvert_Rint2_sat_rtn(long2); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtz(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rte(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtp(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtn(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtz(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rte(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtp(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtn(char3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtz(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rte(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtp(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtn(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtz(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rte(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtp(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtn(short3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtz(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rte(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtp(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtn(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtz(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rte(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtp(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtn(int3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtz(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rte(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtp(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_rtn(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtz(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rte(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtp(long3); __attribute__((overloadable)) int3 __spirv_SConvert_Rint3_sat_rtn(long3); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtz(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rte(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtp(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtn(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtz(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rte(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtp(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtn(char4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtz(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rte(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtp(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtn(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtz(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rte(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtp(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtn(short4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtz(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rte(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtp(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtn(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtz(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rte(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtp(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtn(int4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtz(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rte(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtp(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_rtn(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtz(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rte(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtp(long4); __attribute__((overloadable)) int4 __spirv_SConvert_Rint4_sat_rtn(long4); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtz(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rte(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtp(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtn(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtz(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rte(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtp(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtn(char8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtz(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rte(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtp(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtn(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtz(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rte(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtp(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtn(short8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtz(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rte(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtp(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtn(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtz(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rte(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtp(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtn(int8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtz(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rte(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtp(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_rtn(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtz(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rte(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtp(long8); __attribute__((overloadable)) int8 __spirv_SConvert_Rint8_sat_rtn(long8); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtz(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rte(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtp(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtn(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtz(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rte(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtp(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtn(char16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtz(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rte(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtp(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtn(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtz(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rte(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtp(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtn(short16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtz(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rte(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtp(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtn(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtz(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rte(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtp(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtn(int16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtz(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rte(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtp(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_rtn(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtz(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rte(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtp(long16); __attribute__((overloadable)) int16 __spirv_SConvert_Rint16_sat_rtn(long16); __attribute__((overloadable)) long __spirv_SConvert_Rlong(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtz(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rte(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtp(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtn(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtz(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rte(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtp(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtn(char); __attribute__((overloadable)) long __spirv_SConvert_Rlong(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtz(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rte(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtp(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtn(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtz(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rte(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtp(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtn(short); __attribute__((overloadable)) long __spirv_SConvert_Rlong(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtz(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rte(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtp(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtn(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtz(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rte(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtp(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtn(int); __attribute__((overloadable)) long __spirv_SConvert_Rlong(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtz(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rte(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtp(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_rtn(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtz(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rte(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtp(long); __attribute__((overloadable)) long __spirv_SConvert_Rlong_sat_rtn(long); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtz(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rte(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtp(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtn(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtz(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rte(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtp(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtn(char2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtz(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rte(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtp(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtn(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtz(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rte(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtp(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtn(short2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtz(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rte(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtp(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtn(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtz(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rte(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtp(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtn(int2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtz(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rte(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtp(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_rtn(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtz(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rte(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtp(long2); __attribute__((overloadable)) long2 __spirv_SConvert_Rlong2_sat_rtn(long2); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtz(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rte(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtp(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtn(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtz(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rte(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtp(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtn(char3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtz(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rte(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtp(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtn(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtz(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rte(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtp(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtn(short3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtz(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rte(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtp(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtn(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtz(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rte(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtp(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtn(int3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtz(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rte(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtp(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_rtn(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtz(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rte(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtp(long3); __attribute__((overloadable)) long3 __spirv_SConvert_Rlong3_sat_rtn(long3); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtz(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rte(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtp(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtn(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtz(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rte(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtp(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtn(char4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtz(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rte(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtp(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtn(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtz(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rte(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtp(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtn(short4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtz(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rte(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtp(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtn(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtz(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rte(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtp(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtn(int4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtz(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rte(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtp(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_rtn(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtz(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rte(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtp(long4); __attribute__((overloadable)) long4 __spirv_SConvert_Rlong4_sat_rtn(long4); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtz(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rte(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtp(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtn(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtz(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rte(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtp(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtn(char8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtz(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rte(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtp(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtn(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtz(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rte(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtp(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtn(short8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtz(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rte(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtp(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtn(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtz(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rte(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtp(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtn(int8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtz(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rte(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtp(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_rtn(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtz(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rte(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtp(long8); __attribute__((overloadable)) long8 __spirv_SConvert_Rlong8_sat_rtn(long8); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtz(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rte(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtp(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtn(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtz(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rte(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtp(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtn(char16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtz(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rte(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtp(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtn(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtz(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rte(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtp(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtn(short16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtz(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rte(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtp(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtn(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtz(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rte(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtp(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtn(int16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtz(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rte(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtp(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_rtn(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtz(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rte(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtp(long16); __attribute__((overloadable)) long16 __spirv_SConvert_Rlong16_sat_rtn(long16); __attribute__((overloadable)) half __spirv_FConvert_Rhalf(half); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtz(half); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rte(half); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtp(half); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtn(half); __attribute__((overloadable)) half __spirv_FConvert_Rhalf(float); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtz(float); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rte(float); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtp(float); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtn(float); __attribute__((overloadable)) half __spirv_FConvert_Rhalf(double); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtz(double); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rte(double); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtp(double); __attribute__((overloadable)) half __spirv_FConvert_Rhalf_rtn(double); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2(half2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtz(half2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rte(half2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtp(half2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtn(half2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2(float2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtz(float2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rte(float2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtp(float2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtn(float2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2(double2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtz(double2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rte(double2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtp(double2); __attribute__((overloadable)) half2 __spirv_FConvert_Rhalf2_rtn(double2); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3(half3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtz(half3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rte(half3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtp(half3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtn(half3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3(float3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtz(float3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rte(float3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtp(float3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtn(float3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3(double3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtz(double3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rte(double3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtp(double3); __attribute__((overloadable)) half3 __spirv_FConvert_Rhalf3_rtn(double3); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4(half4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtz(half4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rte(half4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtp(half4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtn(half4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4(float4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtz(float4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rte(float4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtp(float4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtn(float4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4(double4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtz(double4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rte(double4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtp(double4); __attribute__((overloadable)) half4 __spirv_FConvert_Rhalf4_rtn(double4); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8(half8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtz(half8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rte(half8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtp(half8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtn(half8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8(float8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtz(float8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rte(float8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtp(float8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtn(float8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8(double8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtz(double8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rte(double8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtp(double8); __attribute__((overloadable)) half8 __spirv_FConvert_Rhalf8_rtn(double8); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16(half16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtz(half16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rte(half16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtp(half16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtn(half16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16(float16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtz(float16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rte(float16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtp(float16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtn(float16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16(double16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtz(double16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rte(double16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtp(double16); __attribute__((overloadable)) half16 __spirv_FConvert_Rhalf16_rtn(double16); __attribute__((overloadable)) float __spirv_FConvert_Rfloat(half); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtz(half); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rte(half); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtp(half); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtn(half); __attribute__((overloadable)) float __spirv_FConvert_Rfloat(float); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtz(float); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rte(float); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtp(float); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtn(float); __attribute__((overloadable)) float __spirv_FConvert_Rfloat(double); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtz(double); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rte(double); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtp(double); __attribute__((overloadable)) float __spirv_FConvert_Rfloat_rtn(double); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2(half2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtz(half2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rte(half2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtp(half2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtn(half2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2(float2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtz(float2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rte(float2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtp(float2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtn(float2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2(double2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtz(double2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rte(double2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtp(double2); __attribute__((overloadable)) float2 __spirv_FConvert_Rfloat2_rtn(double2); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3(half3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtz(half3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rte(half3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtp(half3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtn(half3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3(float3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtz(float3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rte(float3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtp(float3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtn(float3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3(double3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtz(double3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rte(double3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtp(double3); __attribute__((overloadable)) float3 __spirv_FConvert_Rfloat3_rtn(double3); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4(half4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtz(half4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rte(half4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtp(half4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtn(half4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4(float4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtz(float4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rte(float4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtp(float4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtn(float4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4(double4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtz(double4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rte(double4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtp(double4); __attribute__((overloadable)) float4 __spirv_FConvert_Rfloat4_rtn(double4); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8(half8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtz(half8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rte(half8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtp(half8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtn(half8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8(float8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtz(float8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rte(float8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtp(float8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtn(float8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8(double8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtz(double8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rte(double8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtp(double8); __attribute__((overloadable)) float8 __spirv_FConvert_Rfloat8_rtn(double8); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16(half16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtz(half16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rte(half16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtp(half16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtn(half16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16(float16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtz(float16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rte(float16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtp(float16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtn(float16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16(double16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtz(double16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rte(double16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtp(double16); __attribute__((overloadable)) float16 __spirv_FConvert_Rfloat16_rtn(double16); __attribute__((overloadable)) double __spirv_FConvert_Rdouble(half); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtz(half); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rte(half); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtp(half); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtn(half); __attribute__((overloadable)) double __spirv_FConvert_Rdouble(float); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtz(float); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rte(float); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtp(float); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtn(float); __attribute__((overloadable)) double __spirv_FConvert_Rdouble(double); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtz(double); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rte(double); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtp(double); __attribute__((overloadable)) double __spirv_FConvert_Rdouble_rtn(double); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2(half2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtz(half2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rte(half2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtp(half2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtn(half2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2(float2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtz(float2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rte(float2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtp(float2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtn(float2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2(double2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtz(double2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rte(double2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtp(double2); __attribute__((overloadable)) double2 __spirv_FConvert_Rdouble2_rtn(double2); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3(half3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtz(half3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rte(half3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtp(half3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtn(half3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3(float3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtz(float3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rte(float3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtp(float3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtn(float3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3(double3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtz(double3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rte(double3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtp(double3); __attribute__((overloadable)) double3 __spirv_FConvert_Rdouble3_rtn(double3); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4(half4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtz(half4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rte(half4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtp(half4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtn(half4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4(float4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtz(float4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rte(float4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtp(float4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtn(float4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4(double4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtz(double4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rte(double4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtp(double4); __attribute__((overloadable)) double4 __spirv_FConvert_Rdouble4_rtn(double4); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8(half8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtz(half8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rte(half8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtp(half8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtn(half8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8(float8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtz(float8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rte(float8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtp(float8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtn(float8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8(double8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtz(double8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rte(double8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtp(double8); __attribute__((overloadable)) double8 __spirv_FConvert_Rdouble8_rtn(double8); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16(half16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtz(half16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rte(half16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtp(half16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtn(half16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16(float16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtz(float16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rte(float16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtp(float16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtn(float16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16(double16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtz(double16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rte(double16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtp(double16); __attribute__((overloadable)) double16 __spirv_FConvert_Rdouble16_rtn(double16); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtz(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rte(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtp(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtn(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtz(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rte(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtp(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtn(char); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtz(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rte(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtp(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtn(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtz(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rte(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtp(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtn(short); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtz(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rte(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtp(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtn(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtz(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rte(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtp(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtn(int); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtz(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rte(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtp(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_rtn(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtz(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rte(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtp(long); __attribute__((overloadable)) uchar __spirv_SatConvertSToU_Ruchar_sat_rtn(long); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtz(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rte(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtp(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtn(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtz(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rte(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtp(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtn(char2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtz(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rte(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtp(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtn(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtz(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rte(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtp(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtn(short2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtz(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rte(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtp(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtn(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtz(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rte(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtp(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtn(int2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtz(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rte(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtp(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_rtn(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtz(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rte(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtp(long2); __attribute__((overloadable)) uchar2 __spirv_SatConvertSToU_Ruchar2_sat_rtn(long2); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtz(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rte(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtp(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtn(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtz(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rte(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtp(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtn(char3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtz(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rte(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtp(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtn(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtz(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rte(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtp(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtn(short3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtz(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rte(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtp(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtn(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtz(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rte(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtp(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtn(int3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtz(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rte(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtp(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_rtn(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtz(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rte(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtp(long3); __attribute__((overloadable)) uchar3 __spirv_SatConvertSToU_Ruchar3_sat_rtn(long3); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtz(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rte(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtp(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtn(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtz(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rte(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtp(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtn(char4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtz(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rte(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtp(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtn(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtz(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rte(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtp(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtn(short4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtz(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rte(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtp(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtn(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtz(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rte(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtp(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtn(int4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtz(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rte(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtp(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_rtn(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtz(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rte(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtp(long4); __attribute__((overloadable)) uchar4 __spirv_SatConvertSToU_Ruchar4_sat_rtn(long4); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtz(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rte(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtp(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtn(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtz(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rte(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtp(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtn(char8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtz(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rte(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtp(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtn(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtz(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rte(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtp(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtn(short8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtz(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rte(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtp(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtn(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtz(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rte(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtp(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtn(int8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtz(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rte(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtp(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_rtn(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtz(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rte(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtp(long8); __attribute__((overloadable)) uchar8 __spirv_SatConvertSToU_Ruchar8_sat_rtn(long8); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtz(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rte(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtp(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtn(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtz(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rte(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtp(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtn(char16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtz(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rte(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtp(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtn(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtz(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rte(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtp(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtn(short16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtz(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rte(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtp(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtn(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtz(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rte(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtp(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtn(int16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtz(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rte(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtp(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_rtn(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtz(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rte(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtp(long16); __attribute__((overloadable)) uchar16 __spirv_SatConvertSToU_Ruchar16_sat_rtn(long16); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtz(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rte(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtp(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtn(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtz(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rte(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtp(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtn(char); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtz(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rte(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtp(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtn(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtz(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rte(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtp(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtn(short); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtz(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rte(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtp(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtn(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtz(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rte(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtp(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtn(int); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtz(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rte(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtp(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_rtn(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtz(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rte(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtp(long); __attribute__((overloadable)) ushort __spirv_SatConvertSToU_Rushort_sat_rtn(long); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtz(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rte(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtp(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtn(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtz(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rte(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtp(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtn(char2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtz(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rte(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtp(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtn(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtz(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rte(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtp(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtn(short2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtz(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rte(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtp(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtn(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtz(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rte(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtp(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtn(int2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtz(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rte(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtp(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_rtn(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtz(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rte(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtp(long2); __attribute__((overloadable)) ushort2 __spirv_SatConvertSToU_Rushort2_sat_rtn(long2); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtz(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rte(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtp(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtn(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtz(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rte(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtp(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtn(char3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtz(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rte(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtp(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtn(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtz(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rte(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtp(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtn(short3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtz(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rte(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtp(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtn(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtz(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rte(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtp(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtn(int3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtz(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rte(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtp(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_rtn(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtz(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rte(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtp(long3); __attribute__((overloadable)) ushort3 __spirv_SatConvertSToU_Rushort3_sat_rtn(long3); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtz(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rte(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtp(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtn(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtz(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rte(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtp(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtn(char4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtz(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rte(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtp(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtn(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtz(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rte(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtp(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtn(short4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtz(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rte(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtp(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtn(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtz(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rte(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtp(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtn(int4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtz(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rte(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtp(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_rtn(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtz(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rte(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtp(long4); __attribute__((overloadable)) ushort4 __spirv_SatConvertSToU_Rushort4_sat_rtn(long4); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtz(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rte(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtp(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtn(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtz(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rte(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtp(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtn(char8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtz(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rte(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtp(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtn(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtz(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rte(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtp(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtn(short8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtz(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rte(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtp(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtn(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtz(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rte(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtp(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtn(int8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtz(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rte(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtp(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_rtn(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtz(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rte(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtp(long8); __attribute__((overloadable)) ushort8 __spirv_SatConvertSToU_Rushort8_sat_rtn(long8); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtz(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rte(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtp(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtn(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtz(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rte(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtp(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtn(char16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtz(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rte(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtp(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtn(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtz(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rte(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtp(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtn(short16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtz(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rte(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtp(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtn(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtz(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rte(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtp(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtn(int16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtz(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rte(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtp(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_rtn(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtz(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rte(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtp(long16); __attribute__((overloadable)) ushort16 __spirv_SatConvertSToU_Rushort16_sat_rtn(long16); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtz(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rte(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtp(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtn(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtz(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rte(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtp(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtn(char); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtz(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rte(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtp(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtn(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtz(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rte(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtp(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtn(short); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtz(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rte(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtp(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtn(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtz(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rte(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtp(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtn(int); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtz(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rte(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtp(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_rtn(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtz(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rte(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtp(long); __attribute__((overloadable)) uint __spirv_SatConvertSToU_Ruint_sat_rtn(long); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtz(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rte(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtp(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtn(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtz(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rte(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtp(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtn(char2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtz(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rte(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtp(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtn(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtz(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rte(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtp(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtn(short2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtz(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rte(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtp(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtn(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtz(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rte(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtp(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtn(int2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtz(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rte(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtp(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_rtn(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtz(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rte(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtp(long2); __attribute__((overloadable)) uint2 __spirv_SatConvertSToU_Ruint2_sat_rtn(long2); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtz(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rte(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtp(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtn(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtz(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rte(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtp(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtn(char3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtz(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rte(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtp(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtn(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtz(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rte(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtp(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtn(short3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtz(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rte(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtp(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtn(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtz(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rte(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtp(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtn(int3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtz(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rte(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtp(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_rtn(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtz(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rte(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtp(long3); __attribute__((overloadable)) uint3 __spirv_SatConvertSToU_Ruint3_sat_rtn(long3); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtz(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rte(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtp(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtn(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtz(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rte(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtp(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtn(char4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtz(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rte(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtp(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtn(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtz(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rte(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtp(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtn(short4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtz(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rte(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtp(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtn(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtz(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rte(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtp(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtn(int4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtz(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rte(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtp(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_rtn(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtz(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rte(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtp(long4); __attribute__((overloadable)) uint4 __spirv_SatConvertSToU_Ruint4_sat_rtn(long4); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtz(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rte(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtp(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtn(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtz(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rte(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtp(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtn(char8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtz(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rte(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtp(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtn(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtz(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rte(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtp(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtn(short8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtz(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rte(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtp(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtn(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtz(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rte(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtp(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtn(int8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtz(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rte(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtp(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_rtn(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtz(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rte(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtp(long8); __attribute__((overloadable)) uint8 __spirv_SatConvertSToU_Ruint8_sat_rtn(long8); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtz(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rte(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtp(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtn(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtz(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rte(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtp(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtn(char16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtz(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rte(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtp(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtn(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtz(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rte(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtp(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtn(short16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtz(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rte(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtp(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtn(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtz(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rte(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtp(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtn(int16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtz(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rte(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtp(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_rtn(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtz(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rte(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtp(long16); __attribute__((overloadable)) uint16 __spirv_SatConvertSToU_Ruint16_sat_rtn(long16); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtz(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rte(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtp(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtn(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtz(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rte(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtp(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtn(char); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtz(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rte(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtp(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtn(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtz(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rte(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtp(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtn(short); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtz(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rte(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtp(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtn(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtz(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rte(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtp(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtn(int); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtz(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rte(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtp(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_rtn(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtz(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rte(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtp(long); __attribute__((overloadable)) ulong __spirv_SatConvertSToU_Rulong_sat_rtn(long); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtz(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rte(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtp(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtn(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtz(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rte(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtp(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtn(char2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtz(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rte(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtp(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtn(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtz(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rte(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtp(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtn(short2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtz(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rte(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtp(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtn(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtz(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rte(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtp(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtn(int2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtz(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rte(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtp(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_rtn(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtz(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rte(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtp(long2); __attribute__((overloadable)) ulong2 __spirv_SatConvertSToU_Rulong2_sat_rtn(long2); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtz(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rte(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtp(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtn(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtz(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rte(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtp(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtn(char3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtz(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rte(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtp(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtn(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtz(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rte(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtp(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtn(short3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtz(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rte(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtp(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtn(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtz(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rte(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtp(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtn(int3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtz(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rte(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtp(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_rtn(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtz(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rte(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtp(long3); __attribute__((overloadable)) ulong3 __spirv_SatConvertSToU_Rulong3_sat_rtn(long3); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtz(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rte(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtp(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtn(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtz(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rte(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtp(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtn(char4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtz(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rte(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtp(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtn(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtz(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rte(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtp(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtn(short4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtz(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rte(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtp(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtn(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtz(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rte(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtp(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtn(int4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtz(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rte(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtp(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_rtn(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtz(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rte(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtp(long4); __attribute__((overloadable)) ulong4 __spirv_SatConvertSToU_Rulong4_sat_rtn(long4); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtz(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rte(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtp(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtn(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtz(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rte(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtp(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtn(char8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtz(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rte(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtp(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtn(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtz(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rte(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtp(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtn(short8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtz(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rte(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtp(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtn(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtz(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rte(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtp(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtn(int8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtz(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rte(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtp(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_rtn(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtz(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rte(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtp(long8); __attribute__((overloadable)) ulong8 __spirv_SatConvertSToU_Rulong8_sat_rtn(long8); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtz(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rte(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtp(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtn(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtz(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rte(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtp(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtn(char16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtz(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rte(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtp(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtn(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtz(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rte(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtp(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtn(short16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtz(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rte(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtp(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtn(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtz(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rte(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtp(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtn(int16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtz(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rte(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtp(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_rtn(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtz(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rte(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtp(long16); __attribute__((overloadable)) ulong16 __spirv_SatConvertSToU_Rulong16_sat_rtn(long16); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtz(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rte(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtp(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtn(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtz(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rte(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtp(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtn(uchar); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtz(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rte(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtp(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtn(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtz(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rte(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtp(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtn(ushort); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtz(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rte(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtp(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtn(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtz(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rte(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtp(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtn(uint); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtz(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rte(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtp(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_rtn(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtz(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rte(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtp(ulong); __attribute__((overloadable)) char __spirv_SatConvertUToS_Rchar_sat_rtn(ulong); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtz(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rte(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtp(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtn(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtz(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rte(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtp(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtn(uchar2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtz(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rte(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtp(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtn(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtz(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rte(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtp(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtn(ushort2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtz(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rte(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtp(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtn(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtz(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rte(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtp(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtn(uint2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtz(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rte(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtp(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_rtn(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtz(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rte(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtp(ulong2); __attribute__((overloadable)) char2 __spirv_SatConvertUToS_Rchar2_sat_rtn(ulong2); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtz(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rte(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtp(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtn(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtz(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rte(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtp(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtn(uchar3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtz(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rte(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtp(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtn(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtz(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rte(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtp(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtn(ushort3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtz(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rte(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtp(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtn(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtz(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rte(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtp(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtn(uint3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtz(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rte(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtp(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_rtn(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtz(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rte(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtp(ulong3); __attribute__((overloadable)) char3 __spirv_SatConvertUToS_Rchar3_sat_rtn(ulong3); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtz(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rte(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtp(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtn(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtz(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rte(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtp(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtn(uchar4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtz(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rte(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtp(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtn(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtz(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rte(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtp(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtn(ushort4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtz(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rte(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtp(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtn(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtz(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rte(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtp(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtn(uint4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtz(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rte(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtp(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_rtn(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtz(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rte(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtp(ulong4); __attribute__((overloadable)) char4 __spirv_SatConvertUToS_Rchar4_sat_rtn(ulong4); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtz(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rte(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtp(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtn(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtz(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rte(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtp(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtn(uchar8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtz(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rte(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtp(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtn(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtz(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rte(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtp(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtn(ushort8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtz(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rte(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtp(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtn(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtz(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rte(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtp(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtn(uint8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtz(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rte(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtp(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_rtn(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtz(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rte(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtp(ulong8); __attribute__((overloadable)) char8 __spirv_SatConvertUToS_Rchar8_sat_rtn(ulong8); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtz(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rte(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtp(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtn(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtz(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rte(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtp(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtn(uchar16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtz(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rte(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtp(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtn(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtz(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rte(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtp(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtn(ushort16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtz(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rte(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtp(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtn(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtz(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rte(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtp(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtn(uint16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtz(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rte(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtp(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_rtn(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtz(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rte(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtp(ulong16); __attribute__((overloadable)) char16 __spirv_SatConvertUToS_Rchar16_sat_rtn(ulong16); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtz(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rte(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtp(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtn(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtz(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rte(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtp(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtn(uchar); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtz(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rte(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtp(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtn(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtz(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rte(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtp(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtn(ushort); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtz(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rte(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtp(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtn(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtz(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rte(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtp(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtn(uint); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtz(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rte(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtp(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_rtn(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtz(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rte(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtp(ulong); __attribute__((overloadable)) short __spirv_SatConvertUToS_Rshort_sat_rtn(ulong); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtz(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rte(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtp(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtn(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtz(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rte(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtp(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtn(uchar2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtz(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rte(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtp(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtn(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtz(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rte(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtp(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtn(ushort2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtz(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rte(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtp(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtn(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtz(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rte(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtp(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtn(uint2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtz(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rte(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtp(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_rtn(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtz(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rte(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtp(ulong2); __attribute__((overloadable)) short2 __spirv_SatConvertUToS_Rshort2_sat_rtn(ulong2); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtz(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rte(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtp(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtn(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtz(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rte(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtp(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtn(uchar3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtz(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rte(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtp(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtn(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtz(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rte(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtp(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtn(ushort3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtz(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rte(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtp(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtn(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtz(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rte(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtp(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtn(uint3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtz(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rte(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtp(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_rtn(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtz(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rte(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtp(ulong3); __attribute__((overloadable)) short3 __spirv_SatConvertUToS_Rshort3_sat_rtn(ulong3); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtz(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rte(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtp(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtn(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtz(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rte(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtp(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtn(uchar4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtz(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rte(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtp(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtn(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtz(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rte(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtp(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtn(ushort4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtz(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rte(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtp(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtn(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtz(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rte(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtp(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtn(uint4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtz(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rte(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtp(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_rtn(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtz(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rte(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtp(ulong4); __attribute__((overloadable)) short4 __spirv_SatConvertUToS_Rshort4_sat_rtn(ulong4); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtz(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rte(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtp(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtn(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtz(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rte(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtp(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtn(uchar8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtz(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rte(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtp(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtn(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtz(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rte(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtp(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtn(ushort8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtz(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rte(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtp(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtn(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtz(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rte(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtp(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtn(uint8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtz(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rte(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtp(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_rtn(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtz(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rte(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtp(ulong8); __attribute__((overloadable)) short8 __spirv_SatConvertUToS_Rshort8_sat_rtn(ulong8); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtz(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rte(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtp(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtn(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtz(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rte(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtp(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtn(uchar16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtz(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rte(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtp(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtn(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtz(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rte(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtp(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtn(ushort16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtz(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rte(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtp(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtn(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtz(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rte(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtp(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtn(uint16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtz(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rte(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtp(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_rtn(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtz(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rte(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtp(ulong16); __attribute__((overloadable)) short16 __spirv_SatConvertUToS_Rshort16_sat_rtn(ulong16); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtz(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rte(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtp(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtn(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtz(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rte(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtp(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtn(uchar); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtz(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rte(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtp(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtn(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtz(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rte(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtp(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtn(ushort); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtz(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rte(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtp(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtn(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtz(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rte(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtp(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtn(uint); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtz(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rte(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtp(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_rtn(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtz(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rte(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtp(ulong); __attribute__((overloadable)) int __spirv_SatConvertUToS_Rint_sat_rtn(ulong); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtz(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rte(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtp(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtn(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtz(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rte(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtp(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtn(uchar2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtz(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rte(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtp(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtn(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtz(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rte(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtp(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtn(ushort2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtz(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rte(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtp(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtn(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtz(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rte(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtp(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtn(uint2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtz(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rte(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtp(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_rtn(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtz(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rte(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtp(ulong2); __attribute__((overloadable)) int2 __spirv_SatConvertUToS_Rint2_sat_rtn(ulong2); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtz(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rte(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtp(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtn(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtz(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rte(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtp(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtn(uchar3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtz(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rte(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtp(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtn(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtz(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rte(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtp(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtn(ushort3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtz(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rte(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtp(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtn(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtz(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rte(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtp(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtn(uint3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtz(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rte(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtp(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_rtn(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtz(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rte(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtp(ulong3); __attribute__((overloadable)) int3 __spirv_SatConvertUToS_Rint3_sat_rtn(ulong3); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtz(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rte(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtp(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtn(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtz(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rte(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtp(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtn(uchar4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtz(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rte(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtp(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtn(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtz(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rte(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtp(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtn(ushort4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtz(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rte(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtp(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtn(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtz(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rte(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtp(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtn(uint4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtz(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rte(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtp(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_rtn(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtz(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rte(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtp(ulong4); __attribute__((overloadable)) int4 __spirv_SatConvertUToS_Rint4_sat_rtn(ulong4); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtz(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rte(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtp(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtn(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtz(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rte(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtp(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtn(uchar8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtz(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rte(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtp(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtn(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtz(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rte(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtp(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtn(ushort8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtz(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rte(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtp(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtn(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtz(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rte(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtp(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtn(uint8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtz(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rte(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtp(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_rtn(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtz(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rte(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtp(ulong8); __attribute__((overloadable)) int8 __spirv_SatConvertUToS_Rint8_sat_rtn(ulong8); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtz(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rte(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtp(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtn(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtz(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rte(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtp(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtn(uchar16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtz(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rte(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtp(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtn(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtz(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rte(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtp(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtn(ushort16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtz(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rte(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtp(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtn(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtz(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rte(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtp(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtn(uint16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtz(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rte(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtp(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_rtn(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtz(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rte(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtp(ulong16); __attribute__((overloadable)) int16 __spirv_SatConvertUToS_Rint16_sat_rtn(ulong16); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtz(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rte(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtp(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtn(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtz(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rte(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtp(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtn(uchar); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtz(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rte(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtp(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtn(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtz(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rte(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtp(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtn(ushort); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtz(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rte(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtp(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtn(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtz(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rte(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtp(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtn(uint); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtz(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rte(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtp(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_rtn(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtz(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rte(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtp(ulong); __attribute__((overloadable)) long __spirv_SatConvertUToS_Rlong_sat_rtn(ulong); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtz(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rte(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtp(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtn(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtz(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rte(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtp(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtn(uchar2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtz(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rte(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtp(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtn(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtz(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rte(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtp(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtn(ushort2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtz(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rte(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtp(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtn(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtz(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rte(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtp(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtn(uint2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtz(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rte(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtp(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_rtn(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtz(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rte(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtp(ulong2); __attribute__((overloadable)) long2 __spirv_SatConvertUToS_Rlong2_sat_rtn(ulong2); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtz(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rte(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtp(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtn(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtz(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rte(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtp(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtn(uchar3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtz(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rte(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtp(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtn(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtz(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rte(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtp(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtn(ushort3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtz(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rte(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtp(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtn(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtz(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rte(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtp(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtn(uint3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtz(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rte(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtp(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_rtn(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtz(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rte(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtp(ulong3); __attribute__((overloadable)) long3 __spirv_SatConvertUToS_Rlong3_sat_rtn(ulong3); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtz(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rte(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtp(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtn(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtz(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rte(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtp(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtn(uchar4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtz(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rte(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtp(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtn(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtz(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rte(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtp(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtn(ushort4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtz(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rte(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtp(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtn(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtz(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rte(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtp(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtn(uint4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtz(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rte(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtp(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_rtn(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtz(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rte(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtp(ulong4); __attribute__((overloadable)) long4 __spirv_SatConvertUToS_Rlong4_sat_rtn(ulong4); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtz(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rte(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtp(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtn(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtz(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rte(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtp(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtn(uchar8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtz(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rte(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtp(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtn(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtz(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rte(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtp(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtn(ushort8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtz(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rte(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtp(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtn(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtz(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rte(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtp(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtn(uint8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtz(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rte(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtp(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_rtn(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtz(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rte(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtp(ulong8); __attribute__((overloadable)) long8 __spirv_SatConvertUToS_Rlong8_sat_rtn(ulong8); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtz(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rte(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtp(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtn(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtz(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rte(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtp(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtn(uchar16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtz(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rte(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtp(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtn(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtz(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rte(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtp(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtn(ushort16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtz(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rte(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtp(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtn(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtz(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rte(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtp(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtn(uint16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtz(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rte(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtp(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_rtn(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtz(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rte(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtp(ulong16); __attribute__((overloadable)) long16 __spirv_SatConvertUToS_Rlong16_sat_rtn(ulong16); SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/src/000077500000000000000000000000001506646577400225565ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/OpenCL/src/ImageQuerySize.cl000066400000000000000000000037551506646577400260130ustar00rootroot00000000000000#include "../inc/spirv.h" __attribute__((overloadable, always_inline)) int __spirv_ImageQuerySize(image1d_buffer_t img) { return get_image_width(img); } __attribute__((overloadable, always_inline)) int __spirv_ImageQuerySizeLod(image1d_t img, int lod) { return get_image_width(img) >> lod; } __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySize(image1d_array_t img) { return (int2)(get_image_width(img), get_image_array_size(img)); } __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySizeLod(image1d_array_t img, int lod) { return (int2)(get_image_width(img) >> lod, get_image_array_size(img) >> lod); } #define DEFINE_SPIRV_ImageQuerySizeLod_2d(ImgTy) \ __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySizeLod(ImgTy img, int lod) { \ return get_image_dim(img) >> lod; \ } #define DEFINE_SPIRV_ImageQuerySizeLod_2darray(ImgTy) \ __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySizeLod(ImgTy img, int lod) { \ return (int3)(get_image_dim(img) >> lod, get_image_array_size(img) >> lod); \ } #define DEFINE_SPIRV_ImageQuerySize_2d(ImgTy) \ __attribute__((overloadable, always_inline)) int2 __spirv_ImageQuerySize(ImgTy img) { \ return get_image_dim(img); \ } #define DEFINE_SPIRV_ImageQuerySize_2darray(ImgTy) \ __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySize(ImgTy img) { \ return (int3)(get_image_dim(img), get_image_array_size(img)); \ } __attribute__((overloadable, always_inline)) int3 __spirv_ImageQuerySizeLod(image3d_t img, int lod) { return get_image_dim(img).xyz >> lod; } DEFINE_SPIRV_ImageQuerySize_2d(image2d_t) DEFINE_SPIRV_ImageQuerySize_2d(image2d_depth_t) DEFINE_SPIRV_ImageQuerySizeLod_2d(image2d_t) DEFINE_SPIRV_ImageQuerySizeLod_2d(image2d_depth_t) DEFINE_SPIRV_ImageQuerySize_2darray(image2d_array_t) DEFINE_SPIRV_ImageQuerySize_2darray(image2d_array_depth_t) DEFINE_SPIRV_ImageQuerySizeLod_2darray(image2d_array_t) DEFINE_SPIRV_ImageQuerySizeLod_2darray(image2d_array_depth_t) SPIRV-LLVM-Translator-15.0.17/lib/SPIRV/runtime/README.txt000066400000000000000000000006521506646577400223500ustar00rootroot00000000000000This directory contains SPIR-V builtin functions used by the LLVM module converted from SPIR-V by the SPIR-V/LLVM converter. The SPIR-V consumers need to add these builtin functions to their runtime library. For OpenCL, most of the SPIR-V instructions are translated to either LLVM instructions or OpenCL builtin function calls by the converter. Therefore only a few SPIR-V instructions need to be implemented in the runtime.SPIRV-LLVM-Translator-15.0.17/spirv-headers-tag.conf000066400000000000000000000000511506646577400217030ustar00rootroot000000000000009e3836d7d6023843a72ecd3fbf3f09b1b6747a9e SPIRV-LLVM-Translator-15.0.17/test/000077500000000000000000000000001506646577400164725ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/AlignmentId.spvasm000066400000000000000000000023671506646577400221300ustar00rootroot00000000000000; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.2 -o %t.spv ; RUN: spirv-val %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s OpCapability Addresses OpCapability Linkage OpCapability Kernel OpMemoryModel Physical64 OpenCL OpEntryPoint Kernel %fn "testAlignmentId" OpName %p "p" OpDecorate %x LinkageAttributes "x" Export OpDecorateId %x AlignmentId %al OpDecorateId %p AlignmentId %al_spec %void = OpTypeVoid %uint = OpTypeInt 32 0 %ptr = OpTypePointer CrossWorkgroup %uint %fnTy = OpTypeFunction %void %ptr %al = OpConstant %uint 16 %al_spec = OpSpecConstantOp %uint IAdd %al %al %uint_42 = OpConstant %uint 42 ; Verify alignment of variable. ; CHECK: @x = addrspace(1) global i32 42, align 16 %x = OpVariable %ptr CrossWorkgroup %uint_42 %fn = OpFunction %void None %fnTy ; Verify alignment of function parameter. ; CHECK: define spir_kernel void @testAlignmentId(i32 addrspace(1)* align 32 %p) %p = OpFunctionParameter %ptr %entry = OpLabel OpReturn OpFunctionEnd SPIRV-LLVM-Translator-15.0.17/test/AtomicBuiltinsFloat.ll000066400000000000000000000104571506646577400227460ustar00rootroot00000000000000; Check that translator generates atomic instructions for atomic builtins ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: spirv-val %t.spv ; CHECK-LABEL: Label ; CHECK: Store ; CHECK-COUNT-3: AtomicStore ; CHECK-COUNT-3: AtomicLoad ; CHECK-COUNT-3: AtomicExchange target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir-unknown-unknown" ; Function Attrs: convergent norecurse nounwind define dso_local spir_kernel void @test_atomic_kernel(float addrspace(3)* %ff) local_unnamed_addr #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %0 = addrspacecast float addrspace(3)* %ff to float addrspace(4)* tail call spir_func void @_Z11atomic_initPU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2 tail call spir_func void @_Z12atomic_storePU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2 tail call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)* %0, float 1.000000e+00, i32 0) #2 tail call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)* %0, float 1.000000e+00, i32 0, i32 1) #2 %call = tail call spir_func float @_Z11atomic_loadPU3AS4VU7_Atomicf(float addrspace(4)* %0) #2 %call1 = tail call spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order(float addrspace(4)* %0, i32 0) #2 %call2 = tail call spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order12memory_scope(float addrspace(4)* %0, i32 0, i32 1) #2 %call3 = tail call spir_func float @_Z15atomic_exchangePU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2 %call4 = tail call spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)* %0, float 1.000000e+00, i32 0) #2 %call5 = tail call spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)* %0, float 1.000000e+00, i32 0, i32 1) #2 ret void } ; Function Attrs: convergent declare spir_func void @_Z11atomic_initPU3AS4VU7_Atomicff(float addrspace(4)*, float) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func void @_Z12atomic_storePU3AS4VU7_Atomicff(float addrspace(4)*, float) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)*, float, i32) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)*, float, i32, i32) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z11atomic_loadPU3AS4VU7_Atomicf(float addrspace(4)*) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order(float addrspace(4)*, i32) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order12memory_scope(float addrspace(4)*, i32, i32) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z15atomic_exchangePU3AS4VU7_Atomicff(float addrspace(4)*, float) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)*, float, i32) local_unnamed_addr #1 ; Function Attrs: convergent declare spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)*, float, i32, i32) local_unnamed_addr #1 attributes #0 = { convergent norecurse nounwind "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" } attributes #1 = { convergent "frame-pointer"="none" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } attributes #2 = { convergent nounwind } !llvm.module.flags = !{!0} !opencl.ocl.version = !{!1} !opencl.spir.version = !{!1} !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{i32 2, i32 0} !2 = !{i32 3} !3 = !{!"none"} !4 = !{!"atomic_float*"} !5 = !{!"_Atomic(float)*"} !6 = !{!"volatile"} SPIRV-LLVM-Translator-15.0.17/test/AtomicCompareExchange.ll000066400000000000000000000073261506646577400232210ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: spirv-val %t.spv ; CHECK-SPIRV: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-SPIRV: Constant [[Int]] [[MemScope_Device:[0-9]+]] 1 ; CHECK-SPIRV: Constant [[Int]] [[MemSemEqual_SeqCst:[0-9]+]] 16 ; CHECK-SPIRV: Constant [[Int]] [[MemSemUnequal_Acquire:[0-9]+]] 2 ; CHECK-SPIRV: Constant [[Int]] [[Constant_456:[0-9]+]] 456 ; CHECK-SPIRV: Constant [[Int]] [[Constant_128:[0-9]+]] 128 ; CHECK-SPIRV: TypeBool [[Bool:[0-9]+]] ; CHECK-SPIRV: TypeStruct [[Struct:[0-9]+]] [[Int]] [[Bool]] ; CHECK-SPIRV: Undef [[Struct]] [[UndefStruct:[0-9]+]] ; CHECK-SPIRV: FunctionParameter {{[0-9]+}} [[Pointer:[0-9]+]] ; CHECK-SPIRV: FunctionParameter {{[0-9]+}} [[Value_ptr:[0-9]+]] ; CHECK-SPIRV: FunctionParameter {{[0-9]+}} [[Comparator:[0-9]+]] ; CHECK-SPIRV: Load [[Int]] [[Value:[0-9]+]] [[Value_ptr]] ; CHECK-SPIRV: AtomicCompareExchange [[Int]] [[Res:[0-9]+]] [[Pointer]] [[MemScope_Device]] ; CHECK-SPIRV-SAME: [[MemSemEqual_SeqCst]] [[MemSemUnequal_Acquire]] [[Value]] [[Comparator]] ; CHECK-SPIRV: IEqual {{[0-9]+}} [[Success:[0-9]+]] [[Res]] [[Comparator]] ; CHECK-SPIRV: CompositeInsert [[Struct]] [[Composite_0:[0-9]+]] [[Res]] [[UndefStruct]] 0 ; CHECK-SPIRV: CompositeInsert [[Struct]] [[Composite_1:[0-9]+]] [[Success]] [[Composite_0]] 1 ; CHECK-SPIRV: CompositeExtract [[Bool]] {{[0-9]+}} [[Composite_1]] 1 target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir" ; Function Attrs: nounwind define dso_local spir_func void @test(i32* %ptr, i32* %value_ptr, i32 %comparator) local_unnamed_addr #0 { entry: %0 = load i32, i32* %value_ptr, align 4 %1 = cmpxchg i32* %ptr, i32 %comparator, i32 %0 seq_cst acquire %2 = extractvalue { i32, i1 } %1, 1 br i1 %2, label %cmpxchg.continue, label %cmpxchg.store_expected cmpxchg.store_expected: ; preds = %entry %3 = extractvalue { i32, i1 } %1, 0 store i32 %3, i32* %value_ptr, align 4 br label %cmpxchg.continue cmpxchg.continue: ; preds = %cmpxchg.store_expected, %entry ret void } ; CHECK-SPIRV: FunctionParameter {{[0-9]+}} [[Ptr:[0-9]+]] ; CHECK-SPIRV: FunctionParameter {{[0-9]+}} [[Store_ptr:[0-9]+]] ; CHECK-SPIRV: AtomicCompareExchange [[Int]] [[Res_1:[0-9]+]] [[Ptr]] [[MemScope_Device]] ; CHECK-SPIRV-SAME: [[MemSemEqual_SeqCst]] [[MemSemUnequal_Acquire]] [[Constant_456]] [[Constant_128]] ; CHECK-SPIRV: IEqual {{[0-9]+}} [[Success_1:[0-9]+]] [[Res_1]] [[Constant_128]] ; CHECK-SPIRV: CompositeInsert [[Struct]] [[Composite:[0-9]+]] [[Res_1]] [[UndefStruct]] 0 ; CHECK-SPIRV: CompositeInsert [[Struct]] [[Composite_1:[0-9]+]] [[Success_1]] [[Composite]] 1 ; CHECK-SPIRV: Store [[Store_ptr]] [[Composite_1]] ; Function Attrs: nounwind define dso_local spir_func void @test2(i32* %ptr, {i32, i1}* %store_ptr) local_unnamed_addr #0 { entry: %0 = cmpxchg i32* %ptr, i32 128, i32 456 seq_cst acquire store { i32, i1 } %0, { i32, i1 }* %store_ptr, align 4 ret void } attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind } !llvm.module.flags = !{!0} !llvm.ident = !{!1} !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git cfebd7774229885e7ec88ae9ef1f4ae819cce1d2)"} SPIRV-LLVM-Translator-15.0.17/test/AtomicCompareExchange_cl20.ll000066400000000000000000000115061506646577400240340ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: spirv-val %t.spv target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir-unknown-unknown" ; CHECK: 3 Source 3 200000 ; Int64Atomics capability must be declared only if atomic builtins have 64-bit integers arguments. ; CHECK-NOT: Capability Int64Atomics ; CHECK: 4 TypeInt [[int:[0-9]+]] 32 0 ; CHECK: Constant [[int]] [[DeviceScope:[0-9]+]] 1 ; CHECK: Constant [[int]] [[SequentiallyConsistent_MS:[0-9]+]] 16 ; CHECK: 4 TypePointer [[int_ptr:[0-9]+]] 8 [[int]] ; CHECK: 2 TypeBool [[bool:[0-9]+]] ; Function Attrs: nounwind define spir_func void @test(i32 addrspace(4)* %object, i32 addrspace(4)* %expected, i32 %desired) #0 { ; CHECK: FunctionParameter [[int_ptr]] [[object:[0-9]+]] ; CHECK: FunctionParameter [[int_ptr]] [[expected:[0-9]+]] ; CHECK: FunctionParameter [[int]] [[desired:[0-9]+]] entry: %object.addr = alloca i32 addrspace(4)*, align 4 %expected.addr = alloca i32 addrspace(4)*, align 4 %desired.addr = alloca i32, align 4 %strong_res = alloca i8, align 1 %res = alloca i8, align 1 %weak_res = alloca i8, align 1 store i32 addrspace(4)* %object, i32 addrspace(4)** %object.addr, align 4 store i32 addrspace(4)* %expected, i32 addrspace(4)** %expected.addr, align 4 store i32 %desired, i32* %desired.addr, align 4 %0 = load i32 addrspace(4)*, i32 addrspace(4)** %object.addr, align 4 %1 = load i32 addrspace(4)*, i32 addrspace(4)** %expected.addr, align 4 %2 = load i32, i32* %desired.addr, align 4 ; CHECK: Store [[object_addr:[0-9]+]] [[object]] ; CHECK: Store [[expected_addr:[0-9]+]] [[expected]] ; CHECK: Store [[desired_addr:[0-9]+]] [[desired]] ; CHECK: Load [[int_ptr]] [[Pointer:[0-9]+]] [[object_addr]] ; CHECK: Load [[int_ptr]] [[exp:[0-9]+]] [[expected_addr]] ; CHECK: Load [[int]] [[Value:[0-9]+]] [[desired_addr]] ; CHECK: Load [[int]] [[Comparator:[0-9]+]] [[exp]] ; CHECK-NEXT: 9 AtomicCompareExchange [[int]] [[Result:[0-9]+]] [[Pointer]] [[DeviceScope]] [[SequentiallyConsistent_MS]] [[SequentiallyConsistent_MS]] [[Value]] [[Comparator]] %call = call spir_func zeroext i1 @_Z30atomic_compare_exchange_strongPVU3AS4U7_AtomiciPU3AS4ii(i32 addrspace(4)* %0, i32 addrspace(4)* %1, i32 %2) ; CHECK-NEXT: Store [[exp]] [[Result]] ; CHECK-NEXT: IEqual [[bool]] [[CallRes:[0-9]+]] [[Result]] [[Comparator]] ; CHECK-NOT: [[Result]] %frombool = zext i1 %call to i8 store i8 %frombool, i8* %strong_res, align 1 %3 = load i8, i8* %strong_res, align 1 %tobool = trunc i8 %3 to i1 %lnot = xor i1 %tobool, true %frombool1 = zext i1 %lnot to i8 store i8 %frombool1, i8* %res, align 1 %4 = load i32 addrspace(4)*, i32 addrspace(4)** %object.addr, align 4 %5 = load i32 addrspace(4)*, i32 addrspace(4)** %expected.addr, align 4 %6 = load i32, i32* %desired.addr, align 4 ; CHECK: Load [[int_ptr]] [[Pointer:[0-9]+]] [[object_addr]] ; CHECK: Load [[int_ptr]] [[exp:[0-9]+]] [[expected_addr]] ; CHECK: Load [[int]] [[Value:[0-9]+]] [[desired_addr]] ; CHECK: Load [[int]] [[ComparatorWeak:[0-9]+]] [[exp]] %call2 = call spir_func zeroext i1 @_Z28atomic_compare_exchange_weakPVU3AS4U7_AtomiciPU3AS4ii(i32 addrspace(4)* %4, i32 addrspace(4)* %5, i32 %6) ; CHECK-NEXT: 9 AtomicCompareExchangeWeak [[int]] [[Result:[0-9]+]] [[Pointer]] [[DeviceScope]] [[SequentiallyConsistent_MS]] [[SequentiallyConsistent_MS]] [[Value]] [[ComparatorWeak]] ; CHECK-NEXT: Store [[exp]] [[Result]] ; CHECK-NEXT: IEqual [[bool]] [[CallRes:[0-9]+]] [[Result]] [[ComparatorWeak]] ; CHECK-NOT: [[Result]] %frombool3 = zext i1 %call2 to i8 store i8 %frombool3, i8* %weak_res, align 1 %7 = load i8, i8* %weak_res, align 1 %tobool4 = trunc i8 %7 to i1 %lnot5 = xor i1 %tobool4, true %frombool6 = zext i1 %lnot5 to i8 store i8 %frombool6, i8* %res, align 1 ret void } declare spir_func zeroext i1 @_Z30atomic_compare_exchange_strongPVU3AS4U7_AtomiciPU3AS4ii(i32 addrspace(4)*, i32 addrspace(4)*, i32) #1 declare spir_func zeroext i1 @_Z28atomic_compare_exchange_weakPVU3AS4U7_AtomiciPU3AS4ii(i32 addrspace(4)*, i32 addrspace(4)*, i32) #1 attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !opencl.enable.FP_CONTRACT = !{} !opencl.spir.version = !{!0} !opencl.ocl.version = !{!1} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !0 = !{i32 1, i32 2} !1 = !{i32 2, i32 0} !2 = !{} SPIRV-LLVM-Translator-15.0.17/test/CMakeLists.txt000066400000000000000000000057221506646577400212400ustar00rootroot00000000000000llvm_canonicalize_cmake_booleans(SPIRV_SKIP_CLANG_BUILD) llvm_canonicalize_cmake_booleans(SPIRV_SKIP_DEBUG_INFO_TESTS) # required by lit.site.cfg.py.in get_target_property(LLVM_SPIRV_DIR llvm-spirv BINARY_DIR) set(LLVM_SPIRV_TEST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) if(SPIRV_TOOLS_FOUND) find_program(SPIRV_TOOLS_SPIRV_AS NAMES spirv-as PATHS ${SPIRV_TOOLS_PREFIX}/bin) if(SPIRV_TOOLS_SPIRV_AS) set(SPIRV_TOOLS_SPIRV_AS_FOUND True) endif() find_program(SPIRV_TOOLS_SPIRV_DIS NAMES spirv-dis PATHS ${SPIRV_TOOLS_PREFIX}/bin) if(SPIRV_TOOLS_SPIRV_DIS) set(SPIRV_TOOLS_SPIRV_DIS_FOUND True) endif() find_program(SPIRV_TOOLS_SPIRV_LINK NAMES spirv-link PATHS ${SPIRV_TOOLS_PREFIX}/bin) if(SPIRV_TOOLS_SPIRV_LINK) set(SPIRV_TOOLS_SPIRV_LINK_FOUND True) endif() find_program(SPIRV_TOOLS_SPIRV_VAL NAMES spirv-val PATHS ${SPIRV_TOOLS_PREFIX}/bin) if(SPIRV_TOOLS_SPIRV_VAL) set(SPIRV_TOOLS_SPIRV_VAL_FOUND True) endif() set(SPIRV_TOOLS_BINDIR "${SPIRV_TOOLS_PREFIX}/bin") endif() if(NOT SPIRV_TOOLS_SPIRV_AS) message(WARNING "spirv-as not found! SPIR-V assembly tests will not be run.") set(SPIRV_TOOLS_SPIRV_AS_FOUND False) endif() if(NOT SPIRV_TOOLS_SPIRV_DIS) message(WARNING "spirv-dis not found! SPIR-V disassembly tests will not be run.") set(SPIRV_TOOLS_SPIRV_DIS_FOUND False) endif() if(NOT SPIRV_TOOLS_SPIRV_LINK) message(WARNING "spirv-link not found! SPIR-V test involving the linker will not be run.") set(SPIRV_TOOLS_SPIRV_LINK_FOUND False) endif() if(NOT SPIRV_TOOLS_SPIRV_VAL) message(WARNING "spirv-val not found! SPIR-V generated for test suite will not be validated.") set(SPIRV_TOOLS_SPIRV_VAL_FOUND False) endif() configure_lit_site_cfg( ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py MAIN_CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py ) if(NOT LLVM_SPIRV_BUILD_EXTERNAL) set(LLVM_SPIRV_TEST_DEPS FileCheck count llvm-as llvm-config llvm-dis not ) if(NOT SPIRV_SKIP_CLANG_BUILD) list(APPEND LLVM_SPIRV_TEST_DEPS clang ) endif(NOT SPIRV_SKIP_CLANG_BUILD) if(NOT SPIRV_SKIP_DEBUG_INFO_TESTS) list(APPEND LLVM_SPIRV_TEST_DEPS llc llvm-dwarfdump llvm-objdump llvm-readelf llvm-readobj ) endif(NOT SPIRV_SKIP_DEBUG_INFO_TESTS) endif(NOT LLVM_SPIRV_BUILD_EXTERNAL) add_lit_testsuite(check-llvm-spirv "Running the LLVM-SPIRV regression tests" ${CMAKE_CURRENT_BINARY_DIR} ARGS --verbose DEPENDS ${LLVM_SPIRV_TEST_DEPS} llvm-spirv ) # to enable a custom test target on cmake below 3.11 # starting with 3.11 "test" is only reserved if ENABLE_TESTING(ON) if(LLVM_SPIRV_BUILD_EXTERNAL) cmake_policy(PUSH) if(POLICY CMP0037 AND ${CMAKE_VERSION} VERSION_LESS "3.11.0") cmake_policy(SET CMP0037 OLD) endif(POLICY CMP0037 AND ${CMAKE_VERSION} VERSION_LESS "3.11.0") add_custom_target(test DEPENDS check-llvm-spirv ) cmake_policy(POP) endif(LLVM_SPIRV_BUILD_EXTERNAL) SPIRV-LLVM-Translator-15.0.17/test/CXX/000077500000000000000000000000001506646577400171345ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/CXX/global-ctor.cl000066400000000000000000000021311506646577400216560ustar00rootroot00000000000000// RUN: %clang_cc1 -cl-std=clc++ -emit-llvm-bc -triple spir -O0 %s -o %t.bc -no-opaque-pointers // RUN: llvm-spirv %t.bc -o %t.spv // RUN: spirv-val %t.spv // RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM // RUN: not llvm-spirv %t.bc --spirv-max-version=1.0 2>&1 | FileCheck %s --check-prefix=CHECK-SPV10 class Something { public: Something(int a) : v(a) {} int v; }; Something g(33); void kernel work(global int *out) { *out = g.v; } // CHECK-SPIRV: EntryPoint 6 [[work:[0-9]+]] "work" // CHECK-SPIRV-NOT: ExecutionMode [[work]] 33 // CHECK-SPIRV: EntryPoint 6 [[ctor:[0-9]+]] "_GLOBAL__sub_I_global_ctor.cl" // CHECK-SPIRV: ExecutionMode [[ctor]] 33 // CHECK-LLVM: llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @[[CTORNAME:_GLOBAL__sub_I[^ ]+]], i8* null } // CHECK-LLVM: define spir_kernel void @[[CTORNAME]] // CHECK-SPV10: Feature requires SPIR-V 1.1 or greater: Initializer/Finalizer Execution Mode SPIRV-LLVM-Translator-15.0.17/test/CheckCapKernelWithoutKernel.ll000066400000000000000000000013621506646577400243540ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o %t ; RUN: FileCheck < %t %s ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: spirv-val %t.spv ; ModuleID = '../rel/CheckCapKernelWithoutKernel.bc' target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" @a = addrspace(2) constant i32 1, align 4 ; CHECK-DAG: {{[0-9]*}} Capability Kernel !opencl.enable.FP_CONTRACT = !{} !opencl.spir.version = !{!0} !opencl.ocl.version = !{!1} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!3} !0 = !{i32 1, i32 2} !1 = !{i32 1, i32 0} !2 = !{} !3 = !{!"clang version 3.6.1 "} SPIRV-LLVM-Translator-15.0.17/test/ComparePointers.cl000066400000000000000000000023161506646577400221260ustar00rootroot00000000000000kernel void test(int global *in, int global *in2) { if (!in) return; if (in == 1) return; if (in > in2) return; if (in < in2) return; } // RUN: %clang_cc1 -triple spir64 -x cl -cl-std=CL2.0 -O0 -emit-llvm-bc %s -o %t.bc -no-opaque-pointers // RUN: llvm-spirv %t.bc --spirv-max-version=1.3 -o %t.spv // RUN: spirv-val %t.spv // RUN: llvm-spirv %t.bc -spirv-text --spirv-max-version=1.3 -o %t.spt // RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv %t.bc -o %t.spv // RUN: spirv-val %t.spv // RUN: llvm-spirv %t.bc -spirv-text -o %t.spt // RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV-14 // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:INotEqual // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:IEqual // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:UGreaterThan // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:ConvertPtrToU // CHECK-SPIRV:ULessThan // CHECK-SPIRV-14: PtrNotEqual // CHECK-SPIRV-14: PtrEqual // CHECK-SPIRV-14:ConvertPtrToU // CHECK-SPIRV-14:ConvertPtrToU // CHECK-SPIRV-14:UGreaterThan // CHECK-SPIRV-14:ConvertPtrToU // CHECK-SPIRV-14:ConvertPtrToU // CHECK-SPIRV-14:ULessThan SPIRV-LLVM-Translator-15.0.17/test/ContractionON.ll000066400000000000000000000077531506646577400215570ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-fp-contract=on ; RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s ; RUN: spirv-val %t.spv ; CHECK: EntryPoint 6 [[K1:[0-9]+]] "kernel_off_1" ; CHECK: EntryPoint 6 [[K2:[0-9]+]] "kernel_off_2" ; CHECK: EntryPoint 6 [[K3:[0-9]+]] "kernel_off_3" ; CHECK: EntryPoint 6 [[K4:[0-9]+]] "kernel_off_4" ; CHECK: EntryPoint 6 [[K5:[0-9]+]] "kernel_off_5" ; CHECK: EntryPoint 6 [[K6:[0-9]+]] "kernel_off_6" ; CHECK: EntryPoint 6 [[K7:[0-9]+]] "kernel_on_7" ; CHECK: ExecutionMode [[K1]] 31 ; CHECK: ExecutionMode [[K2]] 31 ; CHECK: ExecutionMode [[K3]] 31 ; CHECK: ExecutionMode [[K4]] 31 ; CHECK: ExecutionMode [[K5]] 31 ; CHECK: ExecutionMode [[K6]] 31 ; CHECK-NOT: ExecutionMode [[K7]] 31 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64" define float @func_nested_off(float %a, float %b, float %c) { entry: %0 = call float @func_off(float %a, float %b, float %c) ret float %0 } define float @func_off(float %a, float %b, float %c) { entry: %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) %mul = fmul float %0, %b %add = fadd float %mul, %c ret float %add } declare float @llvm.fmuladd.f32(float, float, float) #0 define float @func_on(float %a, float %b, float %c) { entry: %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) ret float %0 } define float @func_mul(float %a, float %b) { entry: %0 = fmul float %a, %b ret float %0 } declare float @func_extern(float, float, float) define spir_kernel void @kernel_off_1(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %mul = fmul float %a, %b %add = fadd float %mul, %c ret void } define spir_kernel void @kernel_off_2(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) %call = call float @func_off(float %0, float %b, float %c) ret void } define spir_kernel void @kernel_off_3(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %call = call float @func_nested_off(float %a, float %b, float %c) ret void } define spir_kernel void @kernel_off_4(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) %call = call float @func_extern(float %0, float %b, float %c) ret void } define spir_kernel void @kernel_off_5(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %mul = call float @func_off(float %a, float %b, float %c) %add = fadd contract float %mul, %c ret void } define spir_kernel void @kernel_off_6(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %mul = call float @func_mul(float %a, float %b) %add = fadd float %mul, %c ret void } define spir_kernel void @kernel_on_7(float %a, float %b, float %c) !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 { entry: %mul = call float @func_mul(float %a, float %b) %add = fadd contract float %mul, %c ret void } attributes #0 = { nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0} !opencl.ocl.version = !{!1} !llvm.ident = !{!2} !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{i32 1, i32 0} !2 = !{!"clang version 11.0.0"} !3 = !{i32 0, i32 0, i32 0} !4 = !{!"none", !"none", !"none"} !5 = !{!"float", !"float", !"float"} !6 = !{!"", !"", !""} SPIRV-LLVM-Translator-15.0.17/test/ContractionOff.ll000066400000000000000000000076061506646577400217520ustar00rootroot00000000000000; Source: ; void kernel k1 (float a, float b, float c) { ; #pragma OPENCL FP_CONTRACT OFF ; float d = a * b + c; ; } ; ; void kernel k2 (float a, float b, float c) { ; float d = a * b + c; ; } ; ; void kernel k3 (float a, float b, float c) { ; float d = a * b; // a * b together with -d in the next statement look ; float e = a * c - d; // exactly like an unfused fmuladd in LLVM IR ; } ; ; IR generated using the following commands: ; clang -cc1 -x cl -emit-llvm -O2 -disable-llvm-passes -triple spir64 1.cl -o 1.ll ; opt -mem2reg 1.ll -S -o 1.o.ll ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv.default ; RUN: llvm-spirv %t.spv.default -to-text -o - | FileCheck %s --check-prefixes CHECK,CHECK-ON ; RUN: spirv-val %t.spv.default ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv.off --spirv-fp-contract=off ; RUN: llvm-spirv %t.spv.off -to-text -o - | FileCheck %s --check-prefixes CHECK,CHECK-OFF ; RUN: spirv-val %t.spv.off ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv.fast --spirv-fp-contract=fast ; RUN: llvm-spirv %t.spv.fast -to-text -o - | FileCheck %s --check-prefixes CHECK,CHECK-FAST ; RUN: spirv-val %t.spv.fast ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv.on --spirv-fp-contract=on ; RUN: llvm-spirv %t.spv.on -to-text -o - | FileCheck %s --check-prefixes CHECK,CHECK-ON ; RUN: spirv-val %t.spv.on ; CHECK: EntryPoint 6 [[K1:[0-9]+]] "k1" ; CHECK: EntryPoint 6 [[K2:[0-9]+]] "k2" ; CHECK: EntryPoint 6 [[K3:[0-9]+]] "k3" ; CHECK-OFF: ExecutionMode [[K1]] 31 ; CHECK-OFF: ExecutionMode [[K2]] 31 ; CHECK-OFF: ExecutionMode [[K3]] 31 ; CHECK-FAST-NOT: ExecutionMode [[K1]] 31 ; CHECK-FAST-NOT: ExecutionMode [[K2]] 31 ; CHECK-FAST-NOT: ExecutionMode [[K3]] 31 ; CHECK-ON: ExecutionMode [[K1]] 31 ; CHECK-ON-NOT: ExecutionMode [[K2]] 31 ; CHECK-ON: ExecutionMode [[K3]] 31 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64" ; Function Attrs: convergent nounwind define spir_kernel void @k1(float %a, float %b, float %c) #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 { entry: %mul = fmul float %a, %b %add = fadd float %mul, %c ret void } ; Function Attrs: convergent nounwind define spir_kernel void @k2(float %a, float %b, float %c) #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 { entry: %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c) ret void } ; Function Attrs: nounwind readnone speculatable declare float @llvm.fmuladd.f32(float, float, float) #2 ; Function Attrs: convergent nounwind define spir_kernel void @k3(float %a, float %b, float %c) #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 { entry: %mul = fmul float %a, %b %neg = fsub float -0.000000e+00, %mul %0 = call float @llvm.fmuladd.f32(float %a, float %c, float %neg) ret void } attributes #0 = { convergent nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { argmemonly nounwind } attributes #2 = { nounwind readnone speculatable } !llvm.module.flags = !{!0} !opencl.ocl.version = !{!1} !opencl.spir.version = !{!2} !llvm.ident = !{!3} !0 = !{i32 1, !"wchar_size", i32 4} !1 = !{i32 1, i32 0} !2 = !{i32 1, i32 2} !3 = !{!"clang version 7.0.1"} !4 = !{i32 0, i32 0, i32 0} !5 = !{!"none", !"none", !"none"} !6 = !{!"float", !"float", !"float"} !7 = !{!"", !"", !""} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/000077500000000000000000000000001506646577400203345ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/BuiltinCallLocation.cl000066400000000000000000000014131506646577400245460ustar00rootroot00000000000000// Check that DebugLoc attached to a builtin call is preserved after translation. // RUN: %clang_cc1 -triple spir -fdeclare-opencl-builtins -finclude-default-header %s -disable-llvm-passes -emit-llvm-bc -debug-info-kind=line-tables-only -o %t.bc -no-opaque-pointers // RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv %t.bc -o %t.spv // RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM // CHECK-SPIRV: Label // CHECK-SPIRV: ExtInst {{.*}} DebugScope // CHECK-SPIRV: ExtInst {{.*}} sin // CHECK-LLVM: call spir_func float @_Z3sinf(float %{{.*}}) {{.*}} !dbg ![[loc:[0-9]+]] // CHECK-LLVM: ![[loc]] = !DILocation(line: 14, column: 10, scope: !{{.*}}) float f(float x) { return sin(x); } SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/COFF/000077500000000000000000000000001506646577400210515ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/COFF/global-dllimport.ll000066400000000000000000000036141506646577400246520ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll | FileCheck %s ; CHECK-NOT: S_GDATA32 source_filename = "test/DebugInfo/COFF/global-dllimport.ll" target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32" target triple = "spir64-unknown-unknown" @"\01?id@?$numpunct@D@@0HA" = available_externally dllimport global i32 0, align 4, !dbg !0 !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!14, !15} !llvm.ident = !{!16} !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "id", linkageName: "\01?id@?$numpunct@D@@0HA", scope: !2, file: !6, line: 4, type: !7, isLocal: false, isDefinition: true, declaration: !8) !2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.9.0 (trunk 272628) (llvm/trunk 272566)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5) !3 = !DIFile(filename: "/usr/local/google/home/majnemer/Downloads/", directory: "/usr/local/google/home/majnemer/llvm/src") !4 = !{} !5 = !{!0} !6 = !DIFile(filename: "/usr/local/google/home/majnemer/Downloads/t.ii", directory: "/usr/local/google/home/majnemer/llvm/src") !7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !8 = !DIDerivedType(tag: DW_TAG_member, name: "id", scope: !9, file: !6, line: 2, baseType: !7, flags: DIFlagStaticMember) !9 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "numpunct", file: !6, line: 2, size: 8, align: 8, elements: !10, templateParams: !11) !10 = !{!8} !11 = !{!12} !12 = !DITemplateTypeParameter(type: !13) !13 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !14 = !{i32 2, !"CodeView", i32 1} !15 = !{i32 2, !"Debug Info Version", i32 3} !16 = !{!"clang version 3.9.0 (trunk 272628) (llvm/trunk 272566)"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/COFF/no-cus.ll000066400000000000000000000012671506646577400226140ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -filetype=obj -o %t.o ; RUN: llvm-objdump --section-headers %t.o | FileCheck %s ; Don't emit debug info in this scenario and don't crash. ; CHECK-NOT: .debug$S ; CHECK: .text ; CHECK-NOT: .debug$S ; ModuleID = 't.cpp' source_filename = "t.cpp" target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" define void @f() { entry: ret void } !llvm.module.flags = !{!0, !1} !llvm.ident = !{!2} !0 = !{i32 2, !"CodeView", i32 1} !1 = !{i32 1, !"PIC Level", i32 2} !2 = !{!"clang version 5.0.0 "} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugControlFlow.cl000066400000000000000000000030251506646577400240730ustar00rootroot00000000000000// RUN: %clang_cc1 -triple spir64-unknown-unknown -cl-std=CL2.0 -O0 -debug-info-kind=standalone -gno-column-info -emit-llvm %s -o %t.ll -no-opaque-pointers // RUN: llvm-as -opaque-pointers=0 %t.ll -o %t.bc // RUN: llvm-spirv %t.bc -spirv-text -o %t.spt // RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv %t.bc -o %t.spv // RUN: llvm-spirv -r %t.spv -o %t.bc // RUN: llvm-dis < %t.bc | FileCheck %s --check-prefix=CHECK-LLVM // Test that no debug info instruction is inserted // between LoopMerge and Branch/BranchConditional instructions. // Otherwise, debug info interferes with SPIRVToLLVM translation // of structured flow control kernel void sample() { int arr[10]; #pragma clang loop unroll(enable) for (int i = 0; i < 10; i++) arr[i] = 0; int j = 0; #pragma clang loop unroll(enable) do { arr[j] = 0; } while (j++ < 10); } // Check that all Line items are retained // CHECK-SPIRV: Line [[File:[0-9]+]] 18 0 // Control flow // CHECK-SPIRV: {{[0-9]+}} LoopMerge [[MergeBlock:[0-9]+]] [[ContinueTarget:[0-9]+]] 1 // CHECK-SPIRV-NEXT: BranchConditional // Check that all Line items are retained // CHECK-SPIRV: Line [[File]] 23 0 // CHECK-SPIRV: Line [[File]] 24 0 // Control flow // CHECK-SPIRV: {{[0-9]+}} LoopMerge [[MergeBlock:[0-9]+]] [[ContinueTarget:[0-9]+]] 1 // CHECK-SPIRV-NEXT: Branch // CHECK-LLVM: br label %{{.*}}, !dbg !{{[0-9]+}}, !llvm.loop ![[MD:[0-9]+]] // CHECK-LLVM: ![[MD]] = distinct !{![[MD]], ![[MD_unroll:[0-9]+]]} // CHECK-LLVM: ![[MD_unroll]] = !{!"llvm.loop.unroll.enable"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugDeclareUnused.cl000066400000000000000000000013361506646577400243510ustar00rootroot00000000000000// Check that we can translate llvm.dbg.declare for a local variable which was // deleted by mem2reg pass(disabled by default in llvm-spirv) // RUN: %clang_cc1 %s -triple spir -disable-llvm-passes -debug-info-kind=standalone -emit-llvm-bc -o - -no-opaque-pointers | llvm-spirv -spirv-mem2reg -o %t.spv // RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM void foo() { int a; } // CHECK-SPIRV: Undef [[#]] [[#Undef:]] // CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugDeclare [[#]] [[#Undef]] [[#]] // CHECK-LLVM: call void @llvm.dbg.declare(metadata i32* undef, metadata ![[#]], metadata !DIExpression({{.*}})) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugFunction.cl000066400000000000000000000024321506646577400234110ustar00rootroot00000000000000// Check for 2 things: // - After round trip translation function definition has !dbg metadata attached // specifically if -gline-tables-only was used for Clang // - Parent operand of DebugFunction is DebugCompilationUnit, not an OpString, // even if in LLVM IR it points to a DIFile instead of DICompileUnit. // RUN: %clang_cc1 %s -cl-std=clc++ -emit-llvm-bc -triple spir -debug-info-kind=line-tables-only -O0 -o - -no-opaque-pointers | llvm-spirv -o %t.spv // RUN: llvm-spirv %t.spv -to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM float foo(int i) { return i * 3.14; } void kernel k() { float a = foo(2); } // CHECK-SPIRV: String [[foo:[0-9]+]] "foo" // CHECK-SPIRV: String [[k:[0-9]+]] "k" // CHECK-SPIRV: [[CU:[0-9]+]] {{[0-9]+}} DebugCompilationUnit // CHECK-SPIRV: DebugFunction [[foo]] {{.*}} [[CU]] {{.*}} [[foo_id:[0-9]+]] {{[0-9]+}} {{$}} // CHECK-SPIRV: DebugFunction [[k]] {{.*}} [[CU]] {{.*}} [[k_id:[0-9]+]] {{[0-9]+}} {{$}} // CHECK-SPIRV: Function {{[0-9]+}} [[foo_id]] // CHECK-LLVM: define spir_func float @_Z3fooi(i32 %i) #{{[0-9]+}} !dbg !{{[0-9]+}} { // CHECK-SPIRV: Function {{[0-9]+}} [[k_id]] // CHECK-LLVM: define spir_kernel void @k() #{{[0-9]+}} !dbg !{{[0-9]+}} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfo-GV-with-DIE.spt000066400000000000000000000032111506646577400246050ustar00rootroot00000000000000;; Ensure that reverse translation can handle an Expression in ;; a DebugGlobalVariable's Variable field. ;; This is used to preserve a DIExpressions in a DIGlobalVariableExpression. ; RUN: llvm-spirv -to-binary %s -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll ; CHECK: ![[#]] = !DIGlobalVariableExpression(var: ![[#GV:]], expr: !DIExpression(DW_OP_constu, 1, DW_OP_stack_value)) ; CHECK: ![[#GV]] = distinct !DIGlobalVariable(name: "true", scope: ![[#]], file: ![[#]], line: 3777, type: ![[#]], isLocal: true, isDefinition: true) 119734787 65536 393230 31 0 2 Capability Addresses 2 Capability Linkage 2 Capability Kernel 8 Extension "SPV_KHR_non_semantic_info" 5 ExtInstImport 1 "OpenCL.std" 11 ExtInstImport 2 "NonSemantic.Shader.DebugInfo.100" 3 MemoryModel 2 2 7 String 3 "/path/to/test.cpp" 3 String 6 "0" 3 String 10 "" 4 String 16 "bool" 4 String 23 "true" 3 Source 0 0 4 TypeInt 7 32 0 4 Constant 7 8 1 4 Constant 7 12 65536 4 Constant 7 13 4 4 Constant 7 14 6 4 Constant 7 17 8 4 Constant 7 18 2 4 Constant 7 21 0 4 Constant 7 25 7 4 Constant 7 28 3777 4 Constant 7 29 12 2 TypeVoid 4 6 ExtInst 4 5 2 DebugSource 3 7 ExtInst 4 9 2 DebugBuildIdentifier 6 8 6 ExtInst 4 11 2 DebugStoragePath 10 9 ExtInst 4 15 2 DebugCompilationUnit 12 13 5 14 5 ExtInst 4 19 2 DebugInfoNone 9 ExtInst 4 20 2 DebugTypeBasic 16 17 18 19 7 ExtInst 4 22 2 DebugTypeQualifier 20 21 7 ExtInst 4 24 2 DebugOperation 17 8 6 ExtInst 4 26 2 DebugOperation 25 7 ExtInst 4 27 2 DebugExpression 24 26 14 ExtInst 4 30 2 DebugGlobalVariable 23 22 5 28 21 15 10 27 29 SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoChecksum.ll000066400000000000000000000072461506646577400242230ustar00rootroot00000000000000; Test checks debug info of Checksumkind & Checksum is preserved from LLVM IR to ; SPIR-V and SPIR-V to LLVM IR translation. ; Original .cpp source: ; ; int main() { ; return 0; ; } ; Command line: ; ./clang -cc1 -debug-info-kind=standalone -S -emit-llvm -triple spir -gcodeview -gcodeview-ghash main.cpp ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV-OCL ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o - | FileCheck %s --check-prefix CHECK-SPIRV-200 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; ModuleID = 'source.bc' source_filename = "main.cpp" target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir" ; Function Attrs: noinline norecurse nounwind optnone define i32 @main() #0 !dbg !10 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval, align 4 ret i32 0, !dbg !15 } attributes #0 = { noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5, !6, !7, !8} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!9} ; CHECK-LLVM: !DIFile(filename: "main.cpp" ; CHECK-LLVM-SAME: checksumkind: CSK_MD5, checksum: "7bb56387968a9caa6e9e35fff94eaf7b" ; CHECK-SPIRV-OCL: String [[#REG:]] "//__CSK_MD5:7bb56387968a9caa6e9e35fff94eaf7b" ; CHECK-SPIRV-OCL: DebugSource [[#]] [[#REG]] ; 0 means MD5 ; CHECK-SPIRV-200: String [[#Val:]] "7bb56387968a9caa6e9e35fff94eaf7b" ; CHECK-SPIRV-200: TypeInt [[#TypeInt32:]] 32 ; CHECK-SPIRV-200: Constant [[#TypeInt32]] [[#Kind:]] 0 ; CHECK-SPIRV-200: DebugSource [[#]] [[#Kind]] [[#Val]] !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 7d09e1d7cf27ce781e83f9d388a7a3e1e6487ead)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "oneAPI", checksumkind: CSK_MD5, checksum: "7bb56387968a9caa6e9e35fff94eaf7b") !2 = !{} !3 = !{i32 2, !"CodeView", i32 1} !4 = !{i32 2, !"CodeViewGHash", i32 1} !5 = !{i32 2, !"Debug Info Version", i32 3} !6 = !{i32 1, !"wchar_size", i32 4} !7 = !{i32 1, !"ThinLTO", i32 0} !8 = !{i32 1, !"EnableSplitLTOUnit", i32 1} !9 = !{!"clang version 13.0.0 (https://github.com/llvm/llvm-project.git 7d09e1d7cf27ce781e83f9d388a7a3e1e6487ead)"} !10 = distinct !DISubprogram(name: "main", scope: !11, file: !11, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !11 = !DIFile(filename: "main.cpp", directory: "C:\\", checksumkind: CSK_MD5, checksum: "7bb56387968a9caa6e9e35fff94eaf7b") !12 = !DISubroutineType(types: !13) !13 = !{!14} !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !DILocation(line: 2, column: 3, scope: !10) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoChecksumCompileUnit.ll000066400000000000000000000041411506646577400263630ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV-OCL ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV-200 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefixes=CHECK-LLVM,CHECK-LLVM-200 ; ModuleID = 'array-transform.bc' target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024" target triple = "spir64-unknown-unknown" !llvm.module.flags = !{!0} !llvm.dbg.cu = !{!1} ; CHECK-LLVM: !DIFile(filename: "array-transform.cpp" ; CHECK-LLVM-SAME: checksumkind: CSK_MD5, checksum: "7768106c1e51aa084de0ffae6fbe50c4" ; CHECK-LLVM-200-SAME: source: "int main() {}" ; CHECK-SPIRV-OCL: String [[#ChecksumInfo:]] "//__CSK_MD5:7768106c1e51aa084de0ffae6fbe50c4" ; CHECK-SPIRV-OCL: DebugSource ; CHECK-SPIRV-OCL-SAME: [[#ChecksumInfo]] ; CHECK-SPIRV-200: String [[#Val:]] "7768106c1e51aa084de0ffae6fbe50c4" ; CHECK-SPIRV-200: String [[#Source:]] "int main() {}" ; CHECK-SPIRV-200: TypeInt [[#TypeInt32:]] 32 ; 0 means MD5 ; CHECK-SPIRV-200: Constant [[#TypeInt32]] [[#Kind:]] 0 ; CHECK-SPIRV-200: DebugSource [[#]] [[#Kind]] [[#Val]] [[#Source]] !0 = !{i32 2, !"Debug Info Version", i32 3} !1 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !2, producer: "spirv", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3, imports: !3) !2 = !DIFile(filename: "array-transform.cpp", directory: "D:\\path\\to", checksumkind: CSK_MD5, checksum: "7768106c1e51aa084de0ffae6fbe50c4", source: "int main() {}") !3 = !{} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoGlobalVariableNone.spvasm000066400000000000000000000043711506646577400270450ustar00rootroot00000000000000; Tests translation of GlobalVariable with DebugInfoNone type ; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.1 -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: distinct !DIGlobalVariable(name: "i", linkageName: "_ZL1i", scope: ![[#]], file: ![[#]], line: 1, type: ![[#Type:]], isLocal: true, isDefinition: true) ; CHECK-LLVM: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") ; SPIR-V ; Version: 1.1 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 24 ; Schema: 0 OpCapability Addresses OpCapability Linkage OpCapability Kernel %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "OpenCL.DebugInfo.100" OpMemoryModel Physical64 OpenCL %8 = OpString "/tmp/global.cpp" %13 = OpString "int" %17 = OpString "main" %18 = OpString "" %21 = OpString "i" %22 = OpString "_ZL1i" OpSource Unknown 0 OpName %main "main" OpName %entry "entry" OpModuleProcessed "Debuginfoproducer:clangversion3.4" OpDecorate %main LinkageAttributes "main" Export %uint = OpTypeInt 32 0 %uint_0 = OpConstant %uint 0 %uint_32 = OpConstant %uint 32 %4 = OpTypeFunction %uint %void = OpTypeVoid %10 = OpExtInst %void %2 DebugInfoNone %11 = OpExtInst %void %2 DebugSource %8 %12 = OpExtInst %void %2 DebugCompilationUnit 65536 3 %11 CPP_for_OpenCL %15 = OpExtInst %void %2 DebugTypeBasic %13 %uint_32 Signed %16 = OpExtInst %void %2 DebugTypeFunction None %15 %19 = OpExtInst %void %2 DebugInfoNone %20 = OpExtInst %void %2 DebugFunction %17 %16 %11 2 0 %12 %18 FlagIsDefinition|FlagPrototyped|FlagIsOptimized 2 %main %19 %23 = OpExtInst %void %2 DebugGlobalVariable %21 %10 %11 1 0 %12 %22 %19 FlagIsLocal|FlagIsDefinition %main = OpFunction %uint None %4 %entry = OpLabel %24 = OpExtInst %void %2 DebugScope %20 OpLine %8 4 0 OpReturnValue %uint_0 OpFunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoLLVMArg.ll000066400000000000000000000063141506646577400236600ustar00rootroot00000000000000; This test checks that DW_OP_LLVM_arg operation goes through round trip translation correctly. ; DW_OP_LLVM_arg is mapped on 165 in SPIR-V ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-allow-extra-diexpressions ; RUN: llvm-spirv %t.spv -to-text -o %t.spt ; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-OCL ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv %t.spv -to-text -o %t.spt ; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-200 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; CHECK-SPIRV-200-DAG: TypeInt [[#INT32:]] 32 0 ; CHECK-SPIRV-200-DAG: Constant [[#INT32]] [[#CONST0:]] 0 ; CHECK-SPIRV-200-DAG: Constant [[#INT32]] [[#CONST165:]] 165 ; CHECK-SPIRV: Undef [[#]] [[#UNDEF:]] ; CHECK-SPIRV: [[#DEBUG_LOC_VAR:]] [[#]] DebugLocalVariable ; CHECK-SPIRV-OCL: [[#EXPR_ARG_0:]] [[#]] DebugOperation 165 0 ; CHECK-SPIRV-200: [[#EXPR_ARG_0:]] [[#]] DebugOperation [[#CONST165]] [[#CONST0]] ; CHECK-SPIRV: [[#EXPRESSION:]] [[#]] DebugExpression [[#EXPR_ARG_0]] ; CHECK-SPIRV: [[#EXPR_EMPTY:]] [[#]] DebugExpression{{ *$}} ; CHECK-SPIRV: Variable [[#]] [[#VAL:]] ; CHECK-SPIRV: DebugValue [[#DEBUG_LOC_VAR]] [[#VAL]] [[#EXPRESSION]] ; CHECK-SPIRV: DebugValue [[#DEBUG_LOC_VAR]] [[#UNDEF]] [[#EXPR_EMPTY]] target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone speculatable define void @DbgIntrinsics() sanitize_memtag { entry: %x = alloca i32, align 4 ; CHECK-LLVM: call void @llvm.dbg.value(metadata !DIArgList(ptr %x), metadata ![[#]], metadata !DIExpression(DW_OP_LLVM_arg, 0)) call void @llvm.dbg.value(metadata !DIArgList(i32* %x), metadata !6, metadata !DIExpression(DW_OP_LLVM_arg, 0)), !dbg !10 ; CHECK-LLVM: call void @llvm.dbg.value(metadata ptr undef, metadata ![[#]], metadata !DIExpression()) call void @llvm.dbg.value(metadata !DIArgList(i32* %x, i32* %x), metadata !6, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus)), !dbg !10 store i32 42, i32* %x, align 4 ret void } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "stack-tagging.cc", directory: "/tmp") !2 = !{} !3 = distinct !DISubprogram(name: "DbgIntrinsics", linkageName: "DbgIntrinsics", scope: !1, file: !1, line: 3, type: !4, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !4 = !DISubroutineType(types: !5) !5 = !{null} !6 = !DILocalVariable(name: "x", scope: !3, file: !1, line: 4, type: !7) !7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 2, !"Debug Info Version", i32 3} !10 = !DILocation(line: 1, column: 2, scope: !3) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoLexicalBlockDependency.ll000066400000000000000000000062301506646577400270040ustar00rootroot00000000000000; This test checks the translation of debug info is correct when: ; - Subprogram contains a lexical block LB and a local variable VAL ; - The parent scope of the local variable VAL is the lexical block LB ; - The parent scope of the lexical block LB is the subprogram. ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll target triple = "spir64-unknown-unknown" ; Function Attrs: noinline nounwind define spir_kernel void @test(i64 %value) #0 !dbg !9 { entry: %value.addr = alloca i32, align 1, !dbg !17 call void @llvm.dbg.declare(metadata i32* %value.addr, metadata !13, metadata !DIExpression()), !dbg !18 %value.trunc = trunc i64 %value to i32, !dbg !17 store i32 %value.trunc, i32* %value.addr, align 4, !dbg !17 ret void } ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { noinline nounwind } attributes #1 = { nofree nosync nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !spirv.MemoryModel = !{!5} !spirv.Source = !{!6} !opencl.spir.version = !{!7} !opencl.ocl.version = !{!7} !opencl.used.extensions = !{!4} !opencl.used.optional.core.features = !{!4} !spirv.Generator = !{!8} !0 = !{i32 7, !"Dwarf Version", i32 4} !1 = !{i32 2, !"Debug Info Version", i32 3} !2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "spirv", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, imports: !4) !3 = !DIFile(filename: "1", directory: "/") !4 = !{} !5 = !{i32 2, i32 2} !6 = !{i32 4, i32 200000} !7 = !{i32 2, i32 0} !8 = !{i16 6, i16 14} !9 = distinct !DISubprogram(name: "test", scope: null, file: !3, line: 13, type: !10, scopeLine: 13, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2, templateParams: !4, retainedNodes: !12) !10 = !DISubroutineType(types: !11) !11 = !{null} !12 = !{!13} !13 = !DILocalVariable(name: "value", scope: !14, file: !3, line: 12, type: !15) !14 = distinct !DILexicalBlock(scope: !9, file: !3, line: 13, column: 1) !15 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !16) !16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !17 = !DILocation(line: 13, column: 1, scope: !14) !18 = !DILocation(line: 12, column: 23, scope: !14) ; Debug location metadata id of original instruction. ; CHECK: %value.addr = alloca i32, align 1, !dbg ![[#DL:]] ; The function contains the local variable VAL in retainedNodes. ; CHECK: ![[#SP:]] = distinct !DISubprogram(name: "test" ; CHECK-SAME: retainedNodes: ![[#NODES:]] ; CHECK: ![[#NODES]] = !{![[#VAL:]]} ; The local variable VAL lies in block LB, whose parent scope is the subprogram. ; CHECK: ![[#VAL]] = !DILocalVariable(name: "value", scope: ![[#LB:]] ; CHECK: ![[#LB]] = distinct !DILexicalBlock(scope: ![[#SP]] ; The debug location should be attached to block LB. ; CHECK: ![[#DL]] = !DILocation( ; CHECK-SAME: scope: ![[#LB]] ; Other lexical blocks are unexpected. ; CHECK-NOT: distinct !DILexicalBlock SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoNoneEntity.ll000066400000000000000000000026141506646577400245470ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; Translation shouldn't crash: ; RUN: llvm-spirv %t.bc -spirv-text ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-spirv -spirv-ext=+SPV_INTEL_debug_module %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc source_filename = "llvm-link" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" !llvm.module.flags = !{!1, !2, !3, !4} !llvm.dbg.cu = !{!5} !0 = !{} !1 = !{i32 1, !"wchar_size", i32 4} !2 = !{i32 7, !"PIC Level", i32 2} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 2, !"Dwarf Version", i32 4} !5 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !6, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !0, globals: !0, imports: !7, splitDebugInlining: false, nameTableKind: None) !6 = !DIFile(filename: "declare_target_subroutine.F90", directory: "/test") !7 = !{!8} !8 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !9, entity: !12, file: !6, line: 24) !9 = distinct !DISubprogram(name: "declare_target_subroutine", linkageName: "MAIN__", scope: !6, file: !6, line: 23, type: !10, scopeLine: 23, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !5, retainedNodes: !0) !10 = !DISubroutineType(types: !11) !11 = !{null} !12 = !DIModule(scope: !9, name: "iso_fortran_env", isDecl: true) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoProducer.ll000066400000000000000000000057221506646577400242410ustar00rootroot00000000000000; Test checks debug info of producer is preserved from LLVM IR to spirv ; and spirv to LLVM IR translation. ; Original .cpp source: ; ; int main() { ; return 0; ; } ; Command line: ; ./clang -cc1 -debug-info-kind=standalone -v s.cpp -S -emit-llvm -triple spir ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; ModuleID = 's.bc' source_filename = "s.cpp" target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir" ; Function Attrs: noinline norecurse nounwind optnone define i32 @main() #0 !dbg !8 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval, align 4 ret i32 0, !dbg !13 } attributes #0 = { noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5, !6} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!7} ; CHECK-LLVM: !DICompileUnit ; CHECK-LLVM-SAME: producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-LLVM-NOT: producer: "spirv" ; CHECK-SPIRV: ModuleProcessed "Debug info producer: clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-SPIRV-NOT: DebugBuildIdentifier ; CHECK-SPIRV-NOT: DebugStoragePath !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "oneAPI") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 1, !"ThinLTO", i32 0} !6 = !{i32 1, !"EnableSplitLTOUnit", i32 1} !7 = !{!"clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)"} !8 = distinct !DISubprogram(name: "main", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !9 = !DIFile(filename: "s.cpp", directory: "C:\\") !10 = !DISubroutineType(types: !11) !11 = !{!12} !12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !13 = !DILocation(line: 2, column: 2, scope: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoSubrangeWithOnlyCount.spt000066400000000000000000000041201506646577400271210ustar00rootroot00000000000000; RUN: llvm-spirv -to-binary %s -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s ; CHECK: !DISubrange(count: 1000) 119734787 65792 393230 48 0 2 Capability Addresses 2 Capability Kernel 2 Capability Int64 2 Capability GenericPointer 5 ExtInstImport 1 "OpenCL.std" 5 ExtInstImport 2 "SPIRV.debug" 3 MemoryModel 2 2 15 EntryPoint 6 15 "__omp_offloading_811_198142f_random_fill_sp_l25" 6 String 29 "Fortran/f.f90" 4 String 33 "REAL*4" 16 String 38 "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split" 3 String 39 "" 3 String 41 "a" 3 Source 4 200000 5 Name 4 "structtype" 5 Name 16 "ascast$val" 5 Name 17 "ascastB$val" 5 Name 18 "newFuncRoot" 6 Name 19 "QNCA_a0$float" 5 Name 22 "structtype2" 4 Name 26 ".ascast" 9 ModuleProcessed "Debug info producer: Fortran" 4 Decorate 16 FuncParamAttr 2 4 Decorate 17 FuncParamAttr 4 4 TypeInt 6 64 0 4 TypeInt 10 32 0 5 Constant 6 7 72 0 5 Constant 6 11 1000 0 5 Constant 6 23 1 0 4 Constant 10 34 32 2 TypeVoid 3 2 TypeBool 5 4 TypeArray 8 5 7 3 TypeStruct 4 8 4 TypePointer 9 7 4 4 TypeArray 12 10 11 4 TypePointer 13 5 12 5 TypeFunction 14 3 9 13 3 TypeFloat 20 32 4 TypePointer 21 8 20 5 TypeStruct 22 6 6 6 4 TypeArray 24 22 23 9 TypeStruct 19 21 6 6 6 6 6 24 4 TypePointer 25 7 19 5 ExtInst 3 27 2 DebugInfoNone 7 ExtInst 3 30 2 DebugSource 29 27 9 ExtInst 3 31 2 DebugCompilationUnit 65536 4 30 21 7 ExtInst 3 32 2 DebugTypeFunction 0 27 8 ExtInst 3 35 2 DebugTypeBasic 33 34 3 7 ExtInst 3 36 2 DebugTypeArray 35 11 ; Count = 1000 8 ExtInst 3 37 2 DebugTypePointer 36 4294967295 0 16 ExtInst 3 40 2 DebugFunction 38 32 30 25 0 31 39 44 25 27 27 12 ExtInst 3 42 2 DebugLocalVariable 41 37 30 15 0 40 0 6 ExtInst 3 43 2 DebugTypeTemplate 40 6 ExtInst 3 44 2 DebugOperation 0 6 ExtInst 3 45 2 DebugExpression 44 5 Function 3 15 2 14 3 FunctionParameter 9 16 3 FunctionParameter 13 17 2 Label 18 4 Bitcast 25 26 16 6 ExtInst 3 46 2 DebugScope 43 4 Line 29 15 67 8 ExtInst 3 28 2 DebugValue 42 26 45 5 ExtInst 3 47 2 DebugNoScope 1 Return 1 FunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoTypeBasic.ll000066400000000000000000000061351506646577400243400ustar00rootroot00000000000000; Test checks debug info of basic type is preserved during LLVM IR to spirv ; and spirv to LLVM IR translation. ; Original .cpp source: ; int main() { ; return 0; ; } ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-OCL ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-100 -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-NONSEM ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) ; CHECK-SPIRV: String [[#Name:]] "int" ; CHECK-SPIRV-NONSEM: Constant [[#]] [[#Encoding:]] 4 ; CHECK-SPIRV: Constant [[#]] [[#Size:]] 32 ; CHECK-SPIRV-OCL: DebugTypeBasic [[#Name]] [[#Size]] 4 {{$}} ; CHECK-SPIRV-NONSEM: [[#Flags:]] [[#]] DebugInfoNone ; CHECK-SPIRV-NONSEM: DebugTypeBasic [[#Name]] [[#Size]] [[#Encoding]] [[#Flags]] {{$}} target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: mustprogress noinline nounwind optnone uwtable define dso_local noundef i32 @_Z3foov() #0 !dbg !10 { ret i32 0, !dbg !16 } attributes #0 = { mustprogress noinline nounwind optnone uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} !llvm.ident = !{!9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git b936816fb305cc16d2d1cd6d424c08c39a681a32)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "/app/example.cpp", directory: "/app") !2 = !{i32 7, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 8, !"PIC Level", i32 2} !6 = !{i32 7, !"PIE Level", i32 2} !7 = !{i32 7, !"uwtable", i32 2} !8 = !{i32 7, !"frame-pointer", i32 2} !9 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git b936816fb305cc16d2d1cd6d424c08c39a681a32)"} !10 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !11, file: !11, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !11 = !DIFile(filename: "example.cpp", directory: "/app") !12 = !DISubroutineType(types: !13) !13 = !{!14} !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !{} !16 = !DILocation(line: 1, column: 13, scope: !10)SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoTypeInheritance.ll000066400000000000000000000115271506646577400255510ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-OCL ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-NONSEM ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-SPIRV: String [[#Str_C:]] "C" ; CHECK-SPIRV: String [[#Str_B:]] "B" ; CHECK-SPIRV: String [[#Str_A:]] "A" ; CHECK-SPIRV: [[#Class_A:]] [[#]] DebugTypeComposite [[#Str_A]] ; CHECK-SPIRV-OCL: [[#B_inherits_A:]] [[#]] DebugTypeInheritance [[#Class_B:]] [[#Class_A]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV-NONSEM: [[#B_inherits_A:]] [[#]] DebugTypeInheritance [[#Class_A]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: [[#Class_B:]] [[#]] DebugTypeComposite [[#Str_B]] {{.*}} [[#B_inherits_A]] ; CHECK-SPIRV-OCL: [[#C_inherits_B:]] [[#]] DebugTypeInheritance [[#Class_C:]] [[#Class_B]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV-NONSEM: [[#C_inherits_B:]] [[#]] DebugTypeInheritance [[#Class_B]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: [[#Class_C:]] [[#]] DebugTypeComposite [[#Str_C]] {{.*}} [[#C_inherits_B]] ; CHECK-LLVM: ![[#Class_C:]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "C"{{.*}}identifier: "_ZTS1C") ; CHECK-LLVM: !DIDerivedType(tag: DW_TAG_inheritance, scope: ![[#Class_C]], baseType: ![[#Class_B:]] ; CHECK-LLVM: ![[#Class_B]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "B"{{.*}}identifier: "_ZTS1B") ; CHECK-LLVM: !DIDerivedType(tag: DW_TAG_inheritance, scope: ![[#Class_B]], baseType: ![[#Class_A:]] ; CHECK-LLVM: ![[#Class_A]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "A"{{.*}}identifier: "_ZTS1A") target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" %class.C = type { i8 } ; Function Attrs: mustprogress noinline nounwind optnone uwtable define dso_local noundef i32 @_Z3foov() #0 !dbg !10 { %1 = alloca %class.C, align 1 call void @llvm.dbg.declare(metadata %class.C* %1, metadata !16, metadata !DIExpression()), !dbg !24 ret i32 0, !dbg !25 } ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { mustprogress noinline } attributes #1 = { willreturn } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} !llvm.ident = !{!9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git 1f8a33c19c79fd4649a07eb70ea394c60a8ce316)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "/app/example.cpp", directory: "/app") !2 = !{i32 7, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 8, !"PIC Level", i32 2} !6 = !{i32 7, !"PIE Level", i32 2} !7 = !{i32 7, !"uwtable", i32 2} !8 = !{i32 7, !"frame-pointer", i32 2} !9 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git 1f8a33c19c79fd4649a07eb70ea394c60a8ce316)"} !10 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !11, file: !11, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !11 = !DIFile(filename: "example.cpp", directory: "/app") !12 = !DISubroutineType(types: !13) !13 = !{!14} !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !{} !16 = !DILocalVariable(name: "c", scope: !10, file: !11, line: 7, type: !17) !17 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "C", file: !11, line: 3, size: 8, flags: DIFlagTypePassByValue, elements: !18, identifier: "_ZTS1C") !18 = !{!19} !19 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !17, baseType: !20, flags: DIFlagPublic, extraData: i32 0) !20 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "B", file: !11, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !21, identifier: "_ZTS1B") !21 = !{!22} !22 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !20, baseType: !23, flags: DIFlagPublic, extraData: i32 0) !23 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !11, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !15, identifier: "_ZTS1A") !24 = !DILocation(line: 7, column: 11, scope: !10) !25 = !DILocation(line: 8, column: 3, scope: !10) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoTypeInheritance.spvasm000066400000000000000000000077411506646577400264560ustar00rootroot00000000000000; Tests translation of DebugTypeInheritance and DebugLocalVariable with ; DebugInfoNone type ; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.1 -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: DILocalVariable(name: "c", scope: !9, file: !3, line: 7, type: ![[#Type:]]) ; CHECK-LLVM: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") ; CHECK-LLVM-NOT: DW_TAG_inheritance ; SPIR-V ; Version: 1.0 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 62 ; Schema: 0 OpCapability Addresses OpCapability Linkage OpCapability Kernel OpCapability Int8 OpExtension "SPV_KHR_non_semantic_info" %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "NonSemantic.Shader.DebugInfo.100" OpMemoryModel Physical64 OpenCL %15 = OpString "/app/example.cpp" %17 = OpString "0" %20 = OpString "" %26 = OpString "int" %31 = OpString "_ZTS1C" %32 = OpString "C" %35 = OpString "_ZTS1B" %36 = OpString "B" %38 = OpString "_ZTS1A" %39 = OpString "A" %48 = OpString "foo" %49 = OpString "_Z3foov" %53 = OpString "c" OpSource Unknown 0 OpName %_Z3foov "_Z3foov" OpName %class_C "class.C" OpDecorate %_Z3foov LinkageAttributes "_Z3foov" Export OpDecorate %10 Alignment 1 %uint = OpTypeInt 32 0 %uchar = OpTypeInt 8 0 %uint_0 = OpConstant %uint 0 %uint_1 = OpConstant %uint 1 %uint_65536 = OpConstant %uint 65536 %uint_4 = OpConstant %uint 4 %uint_6 = OpConstant %uint 6 %uint_32 = OpConstant %uint 32 %uint_8 = OpConstant %uint 8 %uint_32768 = OpConstant %uint 32768 %uint_3 = OpConstant %uint 3 %uint_2 = OpConstant %uint 2 %uint_136 = OpConstant %uint 136 %uint_7 = OpConstant %uint 7 %uint_11 = OpConstant %uint 11 %uint_12 = OpConstant %uint 12 %4 = OpTypeFunction %uint %class_C = OpTypeStruct %uchar %_ptr_Function_class_C = OpTypePointer Function %class_C %void = OpTypeVoid %12 = OpExtInst %void %2 DebugInfoNone %16 = OpExtInst %void %2 DebugSource %15 %19 = OpExtInst %void %2 DebugBuildIdentifier %17 %uint_1 %21 = OpExtInst %void %2 DebugStoragePath %20 %25 = OpExtInst %void %2 DebugCompilationUnit %uint_65536 %uint_4 %16 %uint_6 %28 = OpExtInst %void %2 DebugTypeBasic %26 %uint_32 %uint_4 %12 %29 = OpExtInst %void %2 DebugTypeFunction %uint_0 %28 %37 = OpExtInst %void %2 DebugTypeComposite %39 %uint_0 %16 %uint_1 %uint_0 %25 %38 %uint_8 %uint_32768 %43 = OpExtInst %void %2 DebugTypeInheritance %37 %uint_0 %uint_0 %uint_3 %34 = OpExtInst %void %2 DebugTypeComposite %36 %uint_0 %16 %uint_2 %uint_0 %25 %35 %uint_8 %uint_32768 %43 %46 = OpExtInst %void %2 DebugTypeInheritance %34 %uint_0 %uint_0 %uint_3 %30 = OpExtInst %void %2 DebugTypeComposite %32 %uint_0 %16 %uint_3 %uint_0 %25 %31 %uint_8 %uint_32768 %46 %51 = OpExtInst %void %2 DebugFunction %48 %29 %16 %uint_4 %uint_0 %25 %49 %uint_136 %uint_4 %12 %55 = OpExtInst %void %2 DebugLocalVariable %53 %12 %16 %uint_7 %uint_0 %51 %uint_0 %56 = OpExtInst %void %2 DebugExpression %_Z3foov = OpFunction %uint DontInline %4 %6 = OpLabel %52 = OpExtInst %void %2 DebugFunctionDefinition %51 %_Z3foov %10 = OpVariable %_ptr_Function_class_C Function %57 = OpExtInst %void %2 DebugScope %51 %60 = OpExtInst %void %2 DebugLine %15 %uint_7 %uint_7 %uint_11 %uint_12 %13 = OpExtInst %void %2 DebugDeclare %55 %10 %56 %61 = OpExtInst %void %2 DebugLine %15 %uint_8 %uint_8 %uint_3 %uint_4 OpReturnValue %uint_0 OpFunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoTypePtrToMember.spvasm000066400000000000000000000050151506646577400264150ustar00rootroot00000000000000; Tests translation of DebugTypePtrToMember DebugInfoNone type ; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.1 -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: ![[#Type:]] ; CHECK-LLVM: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") ; SPIR-V ; Version: 1.0 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 35 ; Schema: 0 OpCapability Addresses OpCapability Linkage OpCapability Kernel OpCapability Int64 OpExtension "SPV_KHR_non_semantic_info" %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "NonSemantic.Shader.DebugInfo.100" OpMemoryModel Physical64 OpenCL %7 = OpString "./foo.cpp" %10 = OpString "0" %14 = OpString "" %20 = OpString "int" %26 = OpString "_ZTS3Foo" %27 = OpString "Foo" %32 = OpString "x" OpSource Unknown 0 OpName %x "x" OpDecorate %x LinkageAttributes "x" Export OpDecorate %x Alignment 8 %ulong = OpTypeInt 64 0 %uint = OpTypeInt 32 0 %ulong_18446744073709551615 = OpConstant %ulong 18446744073709551615 %uint_1 = OpConstant %uint 1 %uint_65536 = OpConstant %uint 65536 %uint_2 = OpConstant %uint 2 %uint_6 = OpConstant %uint 6 %uint_32 = OpConstant %uint 32 %uint_4 = OpConstant %uint 4 %uint_0 = OpConstant %uint 0 %uint_16 = OpConstant %uint 16 %uint_8 = OpConstant %uint 8 %_ptr_CrossWorkgroup_ulong = OpTypePointer CrossWorkgroup %ulong %void = OpTypeVoid %x = OpVariable %_ptr_CrossWorkgroup_ulong CrossWorkgroup %ulong_18446744073709551615 %9 = OpExtInst %void %2 DebugSource %7 %13 = OpExtInst %void %2 DebugBuildIdentifier %10 %uint_1 %15 = OpExtInst %void %2 DebugStoragePath %14 %19 = OpExtInst %void %2 DebugCompilationUnit %uint_65536 %uint_2 %9 %uint_6 %23 = OpExtInst %void %2 DebugInfoNone %24 = OpExtInst %void %2 DebugTypeBasic %20 %uint_32 %uint_4 %23 %25 = OpExtInst %void %2 DebugTypeComposite %27 %uint_1 %9 %uint_1 %uint_0 %19 %26 %uint_0 %uint_16 %31 = OpExtInst %void %2 DebugTypePtrToMember %23 %25 %34 = OpExtInst %void %2 DebugGlobalVariable %32 %31 %9 %uint_4 %uint_0 %19 %14 %x %uint_8 SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoTypeVector.spvasm000066400000000000000000000127241506646577400254640ustar00rootroot00000000000000; Tests translation of DebugTypeVector DebugInfoNone type ; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.1 -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#Type:]], flags: DIFlagVector ; CHECK-LLVM: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") ; SPIR-V ; Version: 1.1 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 61 ; Schema: 0 OpCapability Addresses OpCapability Linkage OpCapability Kernel OpCapability Int16 %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "OpenCL.DebugInfo.100" OpMemoryModel Physical32 OpenCL OpEntryPoint Kernel %23 "do_add_sub" %__spirv_BuiltInGlobalInvocationId %27 = OpString "/app/" %28 = OpString "//__CSK_MD5:df3c6ff4eef4b9de43419af39216b003" %31 = OpString "short4" %32 = OpString "short" %36 = OpString "/opt/compiler-explorer/clang-assertions-trunk-20240209/lib/clang/19/include/opencl-c-base.h" %37 = OpString "//__CSK_MD5:0cbe46031b64656ef87e922672ce4bdc" %42 = OpString "size_t" %43 = OpString "unsigned int" %47 = OpString "do_add_sub" %48 = OpString "/app/example.cl" %50 = OpString "" %52 = OpString "add_out" %54 = OpString "g" OpSource OpenCL_C 102000 OpName %__spirv_BuiltInGlobalInvocationId "__spirv_BuiltInGlobalInvocationId" OpName %do_add_sub "do_add_sub" OpName %add_out "add_out" OpName %entry "entry" OpName %call "call" OpName %arrayidx "arrayidx" OpName %add_out_0 "add_out" OpModuleProcessed "Debug info producer: clang version 19.0.0git (https://github.com/llvm/llvm-project.git 2572f45c7d6c081ba9b4fa344e928182f8df7773)" OpDecorate %__spirv_BuiltInGlobalInvocationId LinkageAttributes "__spirv_BuiltInGlobalInvocationId" Import OpDecorate %__spirv_BuiltInGlobalInvocationId Constant OpDecorate %__spirv_BuiltInGlobalInvocationId BuiltIn GlobalInvocationId OpDecorate %do_add_sub LinkageAttributes "do_add_sub" Export OpDecorate %add_out FuncParamAttr NoCapture OpDecorate %add_out Alignment 8 OpDecorate %add_out_0 FuncParamAttr NoCapture OpDecorate %add_out_0 Alignment 8 %uint = OpTypeInt 32 0 %ushort = OpTypeInt 16 0 %ushort_2 = OpConstant %ushort 2 %uint_16 = OpConstant %uint 16 %uint_32 = OpConstant %uint 32 %v3uint = OpTypeVector %uint 3 %_ptr_Input_v3uint = OpTypePointer Input %v3uint %void = OpTypeVoid %v4ushort = OpTypeVector %ushort 4 %_ptr_CrossWorkgroup_v4ushort = OpTypePointer CrossWorkgroup %v4ushort %11 = OpTypeFunction %void %_ptr_CrossWorkgroup_v4ushort %__spirv_BuiltInGlobalInvocationId = OpVariable %_ptr_Input_v3uint Input %22 = OpConstantComposite %v4ushort %ushort_2 %ushort_2 %ushort_2 %ushort_2 %15 = OpExtInst %void %2 DebugInfoNone %29 = OpExtInst %void %2 DebugSource %27 %28 %30 = OpExtInst %void %2 DebugCompilationUnit 65536 5 %29 OpenCL_C %34 = OpExtInst %void %2 DebugTypeBasic %32 %uint_16 Signed %35 = OpExtInst %void %2 DebugTypeVector %15 4 %38 = OpExtInst %void %2 DebugSource %36 %37 %39 = OpExtInst %void %2 DebugTypedef %31 %35 %38 0 0 %30 %40 = OpExtInst %void %2 DebugTypePointer %39 CrossWorkgroup None %41 = OpExtInst %void %2 DebugTypeFunction None %15 %40 %45 = OpExtInst %void %2 DebugTypeBasic %43 %uint_32 Unsigned %46 = OpExtInst %void %2 DebugTypedef %42 %45 %38 0 0 %30 %49 = OpExtInst %void %2 DebugSource %48 %28 %51 = OpExtInst %void %2 DebugFunction %47 %41 %49 1 0 %30 %50 FlagIsDefinition|FlagPrototyped|FlagIsOptimized 2 %do_add_sub %15 %53 = OpExtInst %void %2 DebugLocalVariable %52 %40 %49 1 0 %51 None 1 %55 = OpExtInst %void %2 DebugLocalVariable %54 %46 %49 3 0 %51 None %56 = OpExtInst %void %2 DebugOperation Constu 0 %57 = OpExtInst %void %2 DebugOperation Swap %58 = OpExtInst %void %2 DebugOperation Xderef %59 = OpExtInst %void %2 DebugExpression %56 %57 %58 %do_add_sub = OpFunction %void None %11 %add_out = OpFunctionParameter %_ptr_CrossWorkgroup_v4ushort %entry = OpLabel %60 = OpExtInst %void %2 DebugScope %51 OpLine %48 0 0 %16 = OpExtInst %void %2 DebugValue %53 %add_out %59 OpLine %48 3 16 %17 = OpLoad %v3uint %__spirv_BuiltInGlobalInvocationId Aligned 16 %call = OpCompositeExtract %uint %17 0 OpLine %48 0 0 %19 = OpExtInst %void %2 DebugValue %55 %call %59 OpLine %48 4 5 %arrayidx = OpInBoundsPtrAccessChain %_ptr_CrossWorkgroup_v4ushort %add_out %call OpLine %48 4 16 OpStore %arrayidx %22 Aligned 8 OpLine %48 5 1 OpReturn OpFunctionEnd %23 = OpFunction %void None %11 %add_out_0 = OpFunctionParameter %_ptr_CrossWorkgroup_v4ushort %25 = OpLabel %26 = OpFunctionCall %void %do_add_sub %add_out_0 OpReturn OpFunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoVector.ll000066400000000000000000000041501506646577400237120ustar00rootroot00000000000000; Ensure that a vector type's memory size is calculated as bit_ceil(# elements) * element size ; even if the (# elements) is not 3. ; ; This test was derived from DebugInfo/X86/sycl-vec-3.ll. ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv -spirv-ext=+SPV_INTEL_vector_compute ; RUN: llvm-spirv -r %t.spv -o %t.bc ; RUN: llvm-dis %t.bc -o - | FileCheck %s --check-prefixes=CHECK target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" %"class.cl::sycl::vec" = type { <832 x i32> } @vector = dso_local addrspace(1) global %"class.cl::sycl::vec" zeroinitializer, align 16, !dbg !0 !llvm.dbg.cu = !{!9} !llvm.module.flags = !{!10, !11, !12, !13, !14} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "vector", scope: null, file: !2, line: 3, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "sycl-vec-3.cpp", directory: "/tmp") ; CHECK: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[BASE_TY:[0-9]+]],{{.*}} size: 32768, flags: DIFlagVector, elements: ![[ELEMS:[0-9]+]]) !3 = distinct !DICompositeType(tag: DW_TAG_array_type, baseType: !6, file: !2, line: 3, size: 32768, flags: DIFlagVector, elements: !4, identifier: "_ZTSN2cl4sycl3vecIiLi3EEE") ; CHECK-DAG: ![[ELEMS]] = !{![[ELEMS_RANGE:[0-9]+]]} !4 = !{!5} ; CHECK-DAG: ![[ELEMS_RANGE]] = !DISubrange(count: 832{{.*}}) !5 = !DISubrange(count: 832) ; CHECK-DAG: ![[BASE_TY]] = !DIBasicType(name: "int", size: 32,{{.*}} encoding: DW_ATE_signed) !6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !{} !8 = !{!0} !9 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 13.0.0 (https://github.com/intel/llvm.git)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !7, retainedTypes: !7, globals: !8, imports: !7) !10 = !{i32 7, !"Dwarf Version", i32 4} !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{i32 1, !"wchar_size", i32 4} !13 = !{i32 7, !"uwtable", i32 1} !14 = !{i32 7, !"frame-pointer", i32 2} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugInfoWithUnknownIntrinsics.ll000066400000000000000000000056451506646577400270230ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+all --spirv-allow-unknown-intrinsics ; RUN: llvm-spirv -r %t.spv -o %t.bc ; RUN: llvm-dis %t.bc -o %t.ll ; RUN: FileCheck %s --input-file %t.ll -check-prefix=LLVM ; ModuleID = 'test.bc' source_filename = "test.cl" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir" ; Function Attrs: norecurse nounwind readnone uwtable define dso_local i32 @foo(i32 %x, i32 %y) local_unnamed_addr #0 !dbg !8 { entry: call void @llvm.dbg.value(metadata i32 %x, metadata !13, metadata !DIExpression()), !dbg !15 call void @llvm.dbg.value(metadata i32 %y, metadata !14, metadata !DIExpression()), !dbg !15 %add = add nsw i32 %y, %x, !dbg !16 ret i32 %add, !dbg !17 } ; LLVM: declare void @llvm.dbg.value(metadata, metadata, metadata) ; Function Attrs: nounwind readnone speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone speculatable willreturn } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !opencl.ocl.version = !{!6} !llvm.ident = !{!7} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 05b4ff0a4b1a822449e9bf98782b9d337e6f81cf)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "test.cl", directory: "/users/nrudenko/LLVMv/runtests") !2 = !{} !3 = !{i32 7, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{i32 1, i32 0} !7 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 05b4ff0a4b1a822449e9bf98782b9d337e6f81cf)"} !8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) !9 = !DISubroutineType(types: !10) !10 = !{!11, !11, !11} !11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !12 = !{!13, !14} !13 = !DILocalVariable(name: "x", arg: 1, scope: !8, file: !1, line: 1, type: !11) !14 = !DILocalVariable(name: "y", arg: 2, scope: !8, file: !1, line: 1, type: !11) !15 = !DILocation(line: 0, scope: !8) !16 = !DILocation(line: 3, column: 11, scope: !8) !17 = !DILocation(line: 3, column: 3, scope: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugTypeMember.spvasm000066400000000000000000000057541506646577400246220ustar00rootroot00000000000000; Tests translation of DebugTypeMember with DebugInfoNone type ; REQUIRES: spirv-as ; RUN: spirv-as %s --target-env spv1.1 -o %t.spv ; RUN: llvm-spirv -r -o %t.rev.bc %t.spv ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: ![[#Type:]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") ; CHECK-LLVM: !DIDerivedType(tag: DW_TAG_member, name: "anon_static_decl_var", scope: ![[#]], file: ![[#]], line: 5, baseType: ![[#Type]], flags: DIFlagPublic | DIFlagStaticMember, extraData: i32 351) ; SPIR-V ; Version: 1.1 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 36 ; Schema: 0 OpCapability Addresses OpCapability Linkage OpCapability Kernel %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "OpenCL.DebugInfo.100" OpMemoryModel Physical64 OpenCL %8 = OpString "/testdir/test.cpp" %12 = OpString "int" %18 = OpString "anon_static_decl_struct" %19 = OpString "" %23 = OpString "anon_static_decl_var" %28 = OpString "ref" %29 = OpString "_Z3refv" OpSource Unknown 0 OpName %_Z3refv "_Z3refv" OpName %entry "entry" OpModuleProcessed "Debug info producer: clang based Intel(R) oneAPI DPC++/C++ Compiler 2024.1.0 (2024.x.0.YYYYMMDD)" OpDecorate %_Z3refv LinkageAttributes "_Z3refv" Export %uint = OpTypeInt 32 0 %uint_351 = OpConstant %uint 351 %uint_32 = OpConstant %uint 32 %uint_8 = OpConstant %uint 8 %uint_0 = OpConstant %uint 0 %4 = OpTypeFunction %uint %void = OpTypeVoid %10 = OpExtInst %void %2 DebugSource %8 %11 = OpExtInst %void %2 DebugCompilationUnit 65536 4 %10 CPP_for_OpenCL %14 = OpExtInst %void %2 DebugTypeBasic %12 %uint_32 Signed %17 = OpExtInst %void %2 DebugInfoNone %20 = OpExtInst %void %2 DebugSource %19 %21 = OpExtInst %void %2 DebugLexicalBlock %20 0 0 %11 %19 %25 = OpExtInst %void %2 DebugTypeMember %23 %17 %10 5 0 %16 %uint_0 %uint_0 FlagIsProtected|FlagIsPrivate|FlagStaticMember %uint_351 %16 = OpExtInst %void %2 DebugTypeComposite %18 Structure %10 4 0 %21 %17 %uint_8 FlagTypePassByValue %25 %27 = OpExtInst %void %2 DebugTypeFunction None %14 %30 = OpExtInst %void %2 DebugFunction %28 %27 %10 11 0 %11 %29 FlagIsDefinition|FlagPrototyped 11 %_Z3refv %17 %31 = OpExtInst %void %2 DebugOperation Constu 351 %32 = OpExtInst %void %2 DebugOperation StackValue %33 = OpExtInst %void %2 DebugExpression %31 %32 %34 = OpExtInst %void %2 DebugGlobalVariable %23 %17 %10 5 0 %11 %19 %33 FlagIsLocal|FlagIsDefinition %25 %_Z3refv = OpFunction %uint None %4 %entry = OpLabel %35 = OpExtInst %void %2 DebugScope %30 OpLine %8 12 3 OpReturnValue %uint_351 OpFunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/DebugUnstructuredControlFlow.cl000066400000000000000000000021051506646577400265210ustar00rootroot00000000000000// RUN: %clang_cc1 -triple spir64-unknown-unknown -cl-std=CL2.0 -O0 -debug-info-kind=standalone -gno-column-info -emit-llvm-bc %s -o %t.bc -no-opaque-pointers // RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_unstructured_loop_controls -o %t.spv // RUN: llvm-spirv %t.spv --to-text -o %t.spt // RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV // RUN: llvm-spirv -r %t.spv -o %t.bc // RUN: llvm-dis < %t.bc | FileCheck %s --check-prefix=CHECK-LLVM // Test that no debug info instruction is inserted between LoopControlINTEL and // Branch instructions. Otherwise, debug info interferes with SPIRVToLLVM // translation of structured flow control kernel void sample() { #pragma clang loop unroll(full) for(;;); } // Check that all Line items are retained // CHECK-SPIRV: Line [[File:[0-9]+]] 15 0 // Loop control // CHECK-SPIRV: LoopControlINTEL 257 1 // CHECK-SPIRV-NEXT: Branch // CHECK-LLVM: br label %{{.*}}, !dbg !{{[0-9]+}}, !llvm.loop ![[MD:[0-9]+]] // CHECK-LLVM: ![[MD]] = distinct !{![[MD]], ![[MD_unroll:[0-9]+]]} // CHECK-LLVM: ![[MD_unroll]] = !{!"llvm.loop.unroll.full"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/000077500000000000000000000000001506646577400217105ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2009-11-05-DeadGlobalVariable.ll000066400000000000000000000031351506646577400267200ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Here variable bar is optimized away. Do not trip over while trying to generate debug info. source_filename = "test/DebugInfo/Generic/2009-11-05-DeadGlobalVariable.ll" ; Function Attrs: nounwind readnone ssp uwtable define i32 @foo() #0 !dbg !6 { entry: ret i32 42, !dbg !11 } attributes #0 = { nounwind readnone ssp uwtable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !3) !1 = !DIFile(filename: "fb.c", directory: "/private/tmp") !2 = !{} !3 = !{!4} !4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression()) !5 = !DIGlobalVariable(name: "bar", scope: !6, file: !1, line: 2, type: !9, isLocal: true, isDefinition: true) !6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !0) !7 = !DISubroutineType(types: !8) !8 = !{!9} !9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !10 = !{i32 1, !"Debug Info Version", i32 3} !11 = !DILocation(line: 3, column: 3, scope: !12) !12 = distinct !DILexicalBlock(scope: !6, file: !1, line: 1, column: 11) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2009-11-06-NamelessGlobalVariable.ll000066400000000000000000000020711506646577400276310ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" source_filename = "test/DebugInfo/Generic/2009-11-06-NamelessGlobalVariable.ll" @0 = internal constant i32 1, !dbg !0 !llvm.dbg.cu = !{!4} !llvm.module.flags = !{!7} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "a", scope: null, file: !2, line: 2, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "g.c", directory: "/private/tmp") !3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !4 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6) !5 = !{} !6 = !{!0} !7 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2009-11-10-CurrentFn.ll000066400000000000000000000031711506646577400251760ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" define void @bar(i32 %i) nounwind uwtable ssp !dbg !5 { entry: tail call void (...) @foo() nounwind, !dbg !14 ret void, !dbg !16 } declare void @foo(...) declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!18} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 139632)", isOptimized: true, emissionKind: FullDebug, file: !17, enums: !1, retainedTypes: !1, globals: !1) !1 = !{} !5 = distinct !DISubprogram(name: "bar", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, file: !17, scope: !6, type: !7, retainedNodes: !9) !6 = !DIFile(filename: "cf.c", directory: "/private/tmp") !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = !{!11} !11 = !DILocalVariable(name: "i", line: 3, arg: 1, scope: !5, file: !17, type: !12) !12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !13 = !DILocation(line: 3, column: 14, scope: !5) !14 = !DILocation(line: 4, column: 3, scope: !15) !15 = distinct !DILexicalBlock(line: 3, column: 17, file: !17, scope: !5) !16 = !DILocation(line: 5, column: 1, scope: !15) !17 = !DIFile(filename: "cf.c", directory: "/private/tmp") !18 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-01-05-DbgScope.ll000066400000000000000000000026351506646577400247550ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; PR 5942 define i8* @foo() nounwind { entry: %0 = load i32, i32* undef, align 4, !dbg !0 ; [#uses=1] %1 = inttoptr i32 %0 to i8*, !dbg !0 ; [#uses=1] ret i8* %1, !dbg !10 } !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!14} !0 = !DILocation(line: 571, column: 3, scope: !1) !1 = distinct !DILexicalBlock(line: 1, column: 1, file: !11, scope: !2) !2 = distinct !DISubprogram(name: "foo", linkageName: "foo", file: !11, line: 561, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scope: !3, type: !4) !3 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang 1.1", isOptimized: true, emissionKind: FullDebug, file: !11, enums: !12, retainedTypes: !12) !4 = !DISubroutineType(types: !5) !5 = !{!6} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !10 = !DILocation(line: 588, column: 1, scope: !2) !11 = !DIFile(filename: "hashtab.c", directory: "/usr/src/gnu/usr.bin/cc/cc_tools/../../../../contrib/gcclibs/libiberty") !12 = !{} !14 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-03-12-llc-crash.ll000066400000000000000000000027261506646577400251400ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 < %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; llc should not crash on this optimized out debug info. ; PR6588 declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone define void @foo() { entry: call void @llvm.dbg.declare(metadata i32* undef, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1) ret void } !llvm.dbg.cu = !{!3} !0 = !DILocalVariable(name: "sy", line: 890, arg: 1, scope: !1, file: !2, type: !7) !1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 892, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, file: !8, scope: !3, type: !4) !2 = !DIFile(filename: "qpainter.h", directory: "QtGui") !3 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang 1.1", isOptimized: true, emissionKind: FullDebug, file: !9, enums: !10, retainedTypes: !10) !4 = !DISubroutineType(types: !6) !5 = !DIFile(filename: "splineeditor.cpp", directory: "src") !6 = !{null} !7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !8 = !DIFile(filename: "qpainter.h", directory: "QtGui") !9 = !DIFile(filename: "splineeditor.cpp", directory: "src") !10 = !{i32 0} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-03-24-MemberFn.ll000066400000000000000000000104511506646577400247600ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 < %t.ll | grep AT_decl_file | grep 2 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Here _ZN1S3fooEv is defined in header file identified as AT_decl_file no. 2 in debug info. %struct.S = type <{ i8 }> define i32 @_Z3barv() nounwind ssp !dbg !3 { entry: %retval = alloca i32 ; [#uses=2] %0 = alloca i32 ; [#uses=2] %s1 = alloca %struct.S ; <%struct.S*> [#uses=1] %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.declare(metadata %struct.S* %s1, metadata !0, metadata !DIExpression()), !dbg !16 %1 = call i32 @_ZN1S3fooEv(%struct.S* %s1) nounwind, !dbg !17 ; [#uses=1] store i32 %1, i32* %0, align 4, !dbg !17 %2 = load i32, i32* %0, align 4, !dbg !17 ; [#uses=1] store i32 %2, i32* %retval, align 4, !dbg !17 br label %return, !dbg !17 return: ; preds = %entry %retval1 = load i32, i32* %retval, !dbg !17 ; [#uses=1] ret i32 %retval1, !dbg !16 } define linkonce_odr i32 @_ZN1S3fooEv(%struct.S* %this) nounwind ssp align 2 !dbg !12 { entry: %this_addr = alloca %struct.S* ; <%struct.S**> [#uses=1] %retval = alloca i32 ; [#uses=1] %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.declare(metadata %struct.S** %this_addr, metadata !18, metadata !DIExpression(DW_OP_deref)), !dbg !21 store %struct.S* %this, %struct.S** %this_addr br label %return, !dbg !21 return: ; preds = %entry %retval1 = load i32, i32* %retval, !dbg !21 ; [#uses=1] ret i32 %retval1, !dbg !22 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!5} !llvm.module.flags = !{!28} !0 = !DILocalVariable(name: "s1", line: 3, scope: !1, file: !4, type: !9) !1 = distinct !DILexicalBlock(line: 3, column: 0, file: !25, scope: !2) !2 = distinct !DILexicalBlock(line: 3, column: 0, file: !25, scope: !3) !3 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !5, scopeLine: 3, file: !25, scope: !4, type: !6) !4 = !DIFile(filename: "one.cc", directory: "/tmp/") !5 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: FullDebug, file: !25, enums: !27, retainedTypes: !27, imports: null) !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 2, size: 8, align: 8, file: !26, scope: !4, elements: !11) !10 = !DIFile(filename: "one.h", directory: "/tmp/") !11 = !{!12} !12 = distinct !DISubprogram(name: "foo", linkageName: "_ZN1S3fooEv", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !5, scopeLine: 3, file: !26, scope: !9, type: !13) !13 = !DISubroutineType(types: !14) !14 = !{!8, !15} !15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !9) !16 = !DILocation(line: 3, scope: !1) !17 = !DILocation(line: 3, scope: !3) ; Modified from being a pointer, to make this testcase independent of target pointer size !18 = !DILocalVariable(name: "this", line: 3, arg: 1, scope: !12, file: !10, type: !9) !19 = !DIDerivedType(tag: DW_TAG_const_type, size: 64, align: 64, flags: DIFlagArtificial, file: !25, scope: !4, baseType: !20) !20 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !25, scope: !4, baseType: !9) !21 = !DILocation(line: 3, scope: !12) !22 = !DILocation(line: 3, scope: !23) !23 = distinct !DILexicalBlock(line: 3, column: 0, file: !26, scope: !12) !25 = !DIFile(filename: "one.cc", directory: "/tmp/") !26 = !DIFile(filename: "one.h", directory: "/tmp/") !27 = !{} !28 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-04-19-FramePtr.ll000066400000000000000000000043041506646577400250120ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -debugger-tune=lldb -asm-verbose -O1 -o - < %t.ll | FileCheck %s ; RUN: llc -mtriple=%triple -debugger-tune=gdb -asm-verbose -O1 -o - < %t.ll | FileCheck %s --check-prefix=DISABLE ; RUN: llc -mtriple=%triple -frame-pointer=all -debugger-tune=lldb -asm-verbose -O1 -o - < %t.ll | FileCheck %s --check-prefix=DISABLE target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: DW_AT_APPLE_omit_frame_ptr ; DISABLE-NOT: DW_AT_APPLE_omit_frame_ptr define i32 @foo() nounwind ssp !dbg !1 { entry: %retval = alloca i32 ; [#uses=2] %0 = alloca i32 ; [#uses=2] %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] store i32 42, i32* %0, align 4, !dbg !0 %1 = load i32, i32* %0, align 4, !dbg !0 ; [#uses=1] store i32 %1, i32* %retval, align 4, !dbg !0 br label %return, !dbg !0 return: ; preds = %entry %retval1 = load i32, i32* %retval, !dbg !0 ; [#uses=1] ret i32 %retval1, !dbg !7 } !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!12} !9 = !{!1} !0 = !DILocation(line: 2, scope: !1) !1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scopeLine: 2, file: !10, scope: null, type: !4) !2 = !DIFile(filename: "a.c", directory: "/tmp") !3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: FullDebug, file: !10, enums: !11, retainedTypes: !11, imports: null) !4 = !DISubroutineType(types: !5) !5 = !{!6} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !DILocation(line: 2, scope: !8) !8 = distinct !DILexicalBlock(line: 2, column: 0, file: !10, scope: !1) !10 = !DIFile(filename: "a.c", directory: "/tmp") !11 = !{} !12 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-06-29-InlinedFnLocalVar.ll000066400000000000000000000064511506646577400265740ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O2 %t.ll -o - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Check struct X for dead variable xyz from inlined function foo. ; CHECK: debug_info, ; CHECK: DW_TAG_structure_type ; CHECK-NEXT: DW_AT_name source_filename = "test/DebugInfo/Generic/2010-06-29-InlinedFnLocalVar.ll" @i = common global i32 0, !dbg !0 ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #0 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #0 ; Function Attrs: nounwind ssp define i32 @bar() #1 !dbg !8 { entry: %0 = load i32, i32* @i, align 4, !dbg !11 tail call void @llvm.dbg.value(metadata i32 %0, metadata !13, metadata !24), !dbg !25 tail call void @llvm.dbg.declare(metadata !5, metadata !18, metadata !24), !dbg !26 %1 = mul nsw i32 %0, %0, !dbg !27 store i32 %1, i32* @i, align 4, !dbg !11 ret i32 %1, !dbg !28 } attributes #0 = { nounwind readnone } attributes #1 = { nounwind ssp } !llvm.dbg.cu = !{!4} !llvm.module.flags = !{!7} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "i", scope: !2, file: !2, line: 5, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "bar.c", directory: "/tmp/") !3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !4 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6, imports: !5) !5 = !{} !6 = !{!0} !7 = !{i32 1, !"Debug Info Version", i32 3} !8 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: !2, file: !2, line: 14, type: !9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !4) !9 = !DISubroutineType(types: !10) !10 = !{!3} !11 = !DILocation(line: 15, scope: !12) !12 = distinct !DILexicalBlock(scope: !8, file: !2, line: 14) !13 = !DILocalVariable(name: "j", arg: 1, scope: !14, file: !2, line: 9, type: !3) !14 = distinct !DISubprogram(name: "foo", scope: !2, file: !2, line: 9, type: !15, isLocal: true, isDefinition: true, scopeLine: 9, virtualIndex: 6, isOptimized: true, unit: !4, retainedNodes: !17) !15 = !DISubroutineType(types: !16) !16 = !{!3, !3} !17 = !{!13, !18} !18 = !DILocalVariable(name: "xyz", scope: !19, file: !2, line: 10, type: !20) !19 = distinct !DILexicalBlock(scope: !14, file: !2, line: 9) !20 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !14, file: !2, line: 10, size: 64, align: 32, elements: !21) !21 = !{!22, !23} !22 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !20, file: !2, line: 10, baseType: !3, size: 32, align: 32) !23 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !20, file: !2, line: 10, baseType: !3, size: 32, align: 32, offset: 32) !24 = !DIExpression() !25 = !DILocation(line: 9, scope: !14, inlinedAt: !11) !26 = !DILocation(line: 9, scope: !19, inlinedAt: !11) !27 = !DILocation(line: 11, scope: !19, inlinedAt: !11) !28 = !DILocation(line: 16, scope: !12) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/2010-10-01-crash.ll000066400000000000000000000032601506646577400243560ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 %t.ll -o /dev/null target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" define void @CGRectStandardize(i32* sret(i32) %agg.result, i32* byval(i32) %rect) nounwind ssp !dbg !0 { entry: call void @llvm.dbg.declare(metadata i32* %rect, metadata !23, metadata !DIExpression()), !dbg !24 ret void } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!27} !0 = distinct !DISubprogram(name: "CGRectStandardize", linkageName: "CGRectStandardize", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2, file: !1, scope: null, type: !28) !1 = !DIFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop") !2 = distinct !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 2.9 (trunk 115292)", isOptimized: true, runtimeVersion: 1, emissionKind: FullDebug, file: !25, enums: !26, retainedTypes: !26) !5 = !DIDerivedType(tag: DW_TAG_typedef, name: "CGRect", line: 49, file: !25, baseType: null) !23 = !DILocalVariable(name: "rect", line: 53, arg: 2, scope: !0, file: !1, type: !5) !24 = !DILocation(line: 53, column: 33, scope: !0) !25 = !DIFile(filename: "GSFusedSilica.m", directory: "/Volumes/Data/Users/sabre/Desktop") !26 = !{} !27 = !{i32 1, !"Debug Info Version", i32 3} !28 = !DISubroutineType(types: !29) !29 = !{null} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/PR20038.ll000066400000000000000000000210121506646577400231530ustar00rootroot00000000000000; REQUIRES: object-emission ; For some reason, the output when targetting sparc is not quite as expected. ; XFAIL: sparc ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj -dwarf-linkage-names=All < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; IR generated from clang -O0 with: ; struct C { ; ~C(); ; }; ; extern bool b; ; void fun4() { b && (C(), 1); } ; __attribute__((always_inline)) C::~C() { } ; CHECK: DW_TAG_structure_type ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "C" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "~C" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_linkage_name {{.*}} "_ZN1CD1Ev" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "this" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "fun4" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_inlined_subroutine ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "_ZN1CD1Ev" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "this" ; FIXME: D2 is actually inlined into D1 but doesn't show up here, possibly due ; to there being no work in D2 (calling another member function from the dtor ; causes D2 to show up, calling a free function doesn't). ; CHECK-NOT: DW_TAG ; CHECK: NULL ; CHECK-NOT: DW_TAG ; CHECK: NULL %struct.C = type { i8 } @b = external global i8 ; Function Attrs: nounwind define void @_Z4fun4v() #0 !dbg !12 { entry: %this.addr.i.i = alloca %struct.C*, align 8, !dbg !21 %this.addr.i = alloca %struct.C*, align 8, !dbg !22 %agg.tmp.ensured = alloca %struct.C, align 1 %cleanup.cond = alloca i1 %0 = load i8, i8* @b, align 1, !dbg !24 %tobool = trunc i8 %0 to i1, !dbg !24 store i1 false, i1* %cleanup.cond br i1 %tobool, label %land.rhs, label %land.end, !dbg !24 land.rhs: ; preds = %entry store i1 true, i1* %cleanup.cond, !dbg !25 br label %land.end land.end: ; preds = %land.rhs, %entry %1 = phi i1 [ false, %entry ], [ true, %land.rhs ] %cleanup.is_active = load i1, i1* %cleanup.cond, !dbg !27 br i1 %cleanup.is_active, label %cleanup.action, label %cleanup.done, !dbg !27 cleanup.action: ; preds = %land.end store %struct.C* %agg.tmp.ensured, %struct.C** %this.addr.i, align 8, !dbg !22 call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !129, metadata !DIExpression()), !dbg !31 %this1.i = load %struct.C*, %struct.C** %this.addr.i, !dbg !22 store %struct.C* %this1.i, %struct.C** %this.addr.i.i, align 8, !dbg !21 call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i.i, metadata !132, metadata !DIExpression()), !dbg !33 %this1.i.i = load %struct.C*, %struct.C** %this.addr.i.i, !dbg !21 br label %cleanup.done, !dbg !22 cleanup.done: ; preds = %cleanup.action, %land.end ret void, !dbg !34 } ; Function Attrs: alwaysinline nounwind define void @_ZN1CD1Ev(%struct.C* %this) unnamed_addr #1 align 2 !dbg !17 { entry: %this.addr.i = alloca %struct.C*, align 8, !dbg !37 %this.addr = alloca %struct.C*, align 8 store %struct.C* %this, %struct.C** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !29, metadata !DIExpression()), !dbg !38 %this1 = load %struct.C*, %struct.C** %this.addr store %struct.C* %this1, %struct.C** %this.addr.i, align 8, !dbg !37 call void @llvm.dbg.declare(metadata %struct.C** %this.addr.i, metadata !232, metadata !DIExpression()), !dbg !39 %this1.i = load %struct.C*, %struct.C** %this.addr.i, !dbg !37 ret void, !dbg !37 } ; Function Attrs: alwaysinline nounwind define void @_ZN1CD2Ev(%struct.C* %this) unnamed_addr #1 align 2 !dbg !16 { entry: %this.addr = alloca %struct.C*, align 8 store %struct.C* %this, %struct.C** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !32, metadata !DIExpression()), !dbg !40 %this1 = load %struct.C*, %struct.C** %this.addr ret void, !dbg !41 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { alwaysinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!18, !19} !llvm.ident = !{!20} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "", directory: "/tmp/dbginfo") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", line: 1, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS1C") !5 = !DIFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo") !6 = !{!7} !7 = !DISubprogram(name: "~C", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !5, scope: !4, type: !8) !8 = !DISubroutineType(types: !9) !9 = !{null, !10} !10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4) !12 = distinct !DISubprogram(name: "fun4", linkageName: "_Z4fun4v", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 5, file: !5, scope: !13, type: !14, retainedNodes: !2) !13 = !DIFile(filename: "PR20038.cpp", directory: "/tmp/dbginfo") !14 = !DISubroutineType(types: !15) !15 = !{null} !16 = distinct !DISubprogram(name: "~C", linkageName: "_ZN1CD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !5, scope: !4, type: !8, declaration: !7, retainedNodes: !2) !17 = distinct !DISubprogram(name: "~C", linkageName: "_ZN1CD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !5, scope: !4, type: !8, declaration: !7, retainedNodes: !2) !18 = !{i32 2, !"Dwarf Version", i32 4} !19 = !{i32 2, !"Debug Info Version", i32 3} !20 = !{!"clang version 3.5.0 "} !21 = !DILocation(line: 6, scope: !17, inlinedAt: !22) !22 = !DILocation(line: 5, scope: !23) !23 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12) !24 = !DILocation(line: 5, scope: !12) !25 = !DILocation(line: 5, scope: !26) !26 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12) !27 = !DILocation(line: 5, scope: !28) !28 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12) !29 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30) !30 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4) !31 = !DILocation(line: 0, scope: !17, inlinedAt: !22) !32 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30) !33 = !DILocation(line: 0, scope: !16, inlinedAt: !21) !129 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !30) !132 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30) !232 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !16, type: !30) !34 = !DILocation(line: 5, scope: !35) !35 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !36) !36 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !12) !37 = !DILocation(line: 6, scope: !17) !38 = !DILocation(line: 0, scope: !17) !39 = !DILocation(line: 0, scope: !16, inlinedAt: !37) !40 = !DILocation(line: 0, scope: !16) !41 = !DILocation(line: 6, scope: !16) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/bug_null_debuginfo.ll000066400000000000000000000010741506646577400260740ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2} !0 = distinct !DICompileUnit(language: DW_LANG_C99, isOptimized: false, emissionKind: FullDebug, file: !1, globals: null) !1 = !DIFile(filename: "t", directory: "") !2 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/c-and-cpp-mixed.ll000066400000000000000000000176711506646577400251230ustar00rootroot00000000000000;; This test checks that two DICompileUnits resulted in a link of C and C++ ;; object files are being translated correctly ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv --to-text %t.spv -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck < %t.ll %s --check-prefix=CHECK-LLVM ; CHECK-SPIRV: String [[#Foo:]] "foo" ; CHECK-SPIRV: String [[#Main:]] "main" ; CHECK-SPIRV: ExtInst [[#]] [[#CU1:]] [[#]] DebugCompilationUnit ; CHECK-SPIRV: ExtInst [[#]] [[#CU2:]] [[#]] DebugCompilationUnit ; CHECK-SPIRV: ExtInst [[#]] [[#Func1:]] [[#]] DebugFunction [[#Foo]] [[#]] [[#]] [[#]] [[#]] [[#CU1]] ; CHECK-SPIRV: ExtInst [[#]] [[#Func2:]] [[#]] DebugFunction [[#Main]] [[#]] [[#]] [[#]] [[#]] [[#CU2]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugLexicalBlock [[#]] [[#]] [[#]] [[#Func1]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugLexicalBlock [[#]] [[#]] [[#]] [[#Func2]] ; CHECK-LLVM: define spir_func void @foo() #0 !dbg ![[#Func1:]] { ; CHECK-LLVM: entry: ; CHECK-LLVM: %[[#VALUE:]] = getelementptr inbounds [4 x i8], ptr @str, i32 0, i32 0 ; CHECK-LLVM: %puts = call spir_func i32 @puts(ptr nocapture %[[#VALUE]]) #0, !dbg ![[#Puts1Loc:]] ; CHECK-LLVM: ret void, !dbg ![[#Ret1:]] ; CHECK-LLVM: } ; CHECK-LLVM: define spir_func i32 @main(i32 %argc, ptr nocapture %argv) #0 !dbg ![[#Func2:]] { ; CHECK-LLVM: entry: ; CHECK-LLVM: call void @llvm.dbg.value(metadata i32 %argc, metadata ![[#Fun2Param1:]], metadata !DIExpression()), !dbg ![[#Fun2Param1Loc:]] ; CHECK-LLVM: call void @llvm.dbg.value(metadata ptr %argv, metadata ![[#Fun2Param2:]], metadata !DIExpression(DW_OP_deref, DW_OP_deref)), !dbg ![[#Fun2Param2Loc:]] ; CHECK-LLVM: %[[#VALUE1:]] = getelementptr inbounds [6 x i8], ptr @str1, i32 0, i32 0 ; CHECK-LLVM: %puts = call spir_func i32 @puts(ptr nocapture %[[#VALUE1]]) #0, !dbg ![[#Puts2Loc:]] ; CHECK-LLVM: call spir_func void @foo() #0, !dbg ![[#CallFoo:]] ; CHECK-LLVM: ret i32 0, !dbg ![[#Ret2:]] ; CHECK-LLVM: } ; CHECK-LLVM: !llvm.dbg.cu = !{![[#CU1:]], ![[#CU2:]]} ; CHECK-LLVM: ![[#CU1]] = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: ![[#File:]], producer: "clang version 14", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) ; CHECK-LLVM: ![[#File]] = !DIFile(filename: "foo.c", directory: "/tmp") ; CHECK-LLVM: ![[#CU2]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: ![[#File]], producer: "clang version 14", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) ; CHECK-LLVM: ![[#Empty:]] = !{} ; CHECK-LLVM: ![[#Func1]] = distinct !DISubprogram(name: "foo", scope: null, file: ![[#File]], line: 5, type: ![[#Func1T:]], scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[#CU1]], templateParams: ![[#Empty]], retainedNodes: ![[#Empty]]) ; CHECK-LLVM: ![[#Func1T]] = !DISubroutineType(types: ![[#Func1TP:]]) ; CHECK-LLVM: ![[#Func1TP]] = !{null} ; CHECK-LLVM: ![[#Puts1Loc]] = !DILocation(line: 6, column: 3, scope: ![[#Puts1Scope:]]) ; CHECK-LLVM: ![[#Puts1Scope]] = distinct !DILexicalBlock(scope: ![[#Func1]], file: ![[#File]], line: 5, column: 16) ; CHECK-LLVM: ![[#Ret1]] = !DILocation(line: 7, column: 1, scope: ![[#Puts1Scope]]) ; CHECK-LLVM: ![[#Func2]] = distinct !DISubprogram(name: "main", scope: null, file: ![[#File]], line: 11, type: ![[#Func2T:]], scopeLine: 11, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: ![[#CU2]], templateParams: ![[#Empty]], retainedNodes: ![[#Fun2Params:]]) ; CHECK-LLVM: ![[#Func2T]] = !DISubroutineType(types: ![[#Func2TP:]]) ; CHECK-LLVM: ![[#Func2TP]] = !{![[#Func2TP1:]], ; CHECK-LLVM-SAME: ![[#Func2TP1]], ![[#Func2TP2:]] ; CHECK-LLVM: ![[#Func2TP1]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#Func2TP2]] = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) ; CHECK-LLVM: ![[#Fun2Params]] = !{![[#Fun2Param1]], ![[#Fun2Param2]]} ; CHECK-LLVM: ![[#Fun2Param1]] = !DILocalVariable(name: "argc", arg: 1, scope: ![[#Func2]], file: ![[#File]], line: 11, type: ![[#Func2TP1]]) ; CHECK-LLVM: ![[#Fun2Param2]] = !DILocalVariable(name: "argv", arg: 2, scope: ![[#Func2]], file: ![[#File]], line: 11, type: ![[#Func2TP2]]) ; CHECK-LLVM: ![[#Fun2Param1Loc:]] = !DILocation(line: 11, column: 14, scope: ![[#Func2]]) ; CHECK-LLVM: ![[#Fun2Param2Loc:]] = !DILocation(line: 11, column: 26, scope: ![[#Func2]]) ; CHECK-LLVM: ![[#Puts2Loc]] = !DILocation(line: 12, column: 3, scope: ![[#Puts2Scope:]] ; CHECK-LLVM: ![[#Puts2Scope]] = distinct !DILexicalBlock(scope: ![[#Func2]], file: ![[#File]], line: 11, column: 34) ; CHECK-LLVM: ![[#CallFoo]] = !DILocation(line: 13, column: 3, scope: ![[#Puts2Scope]]) ; CHECK-LLVM: ![[#Ret2]] = !DILocation(line: 14, column: 3, scope: ![[#Puts2Scope]]) target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ModuleID = 'test.bc' @str = private unnamed_addr constant [4 x i8] c"FOO\00" @str1 = private unnamed_addr constant [6 x i8] c"Main!\00" define void @foo() nounwind !dbg !5 { entry: %puts = tail call i32 @puts(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @str, i32 0, i32 0)), !dbg !23 ret void, !dbg !25 } declare i32 @puts(i8* nocapture) nounwind define i32 @main(i32 %argc, i8** nocapture %argv) nounwind !dbg !12 { entry: tail call void @llvm.dbg.value(metadata i32 %argc, metadata !21, metadata !DIExpression()), !dbg !26 ; Avoid talking about the pointer size in debug info because that's target dependent tail call void @llvm.dbg.value(metadata i8** %argv, metadata !22, metadata !DIExpression(DW_OP_deref, DW_OP_deref)), !dbg !27 %puts = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str1, i32 0, i32 0)), !dbg !28 tail call void @foo() nounwind, !dbg !30 ret i32 0, !dbg !31 } declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0, !9} !llvm.module.flags = !{!33} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 14", isOptimized: true, emissionKind: FullDebug, file: !32, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !1 = !{} !5 = distinct !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 5, file: !32, scope: !6, type: !7, retainedNodes: !1) !6 = !DIFile(filename: "foo.c", directory: "/tmp") !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, producer: "clang version 14", isOptimized: true, emissionKind: FullDebug, file: !32, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !12 = distinct !DISubprogram(name: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !9, scopeLine: 11, file: !32, scope: !6, type: !13, retainedNodes: !19) !13 = !DISubroutineType(types: !14) !14 = !{!15, !15, !18} !15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !18 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !19 = !{!21, !22} !21 = !DILocalVariable(name: "argc", line: 11, arg: 1, scope: !12, file: !6, type: !15) !22 = !DILocalVariable(name: "argv", line: 11, arg: 2, scope: !12, file: !6, type: !18) !23 = !DILocation(line: 6, column: 3, scope: !24) !24 = distinct !DILexicalBlock(line: 5, column: 16, file: !32, scope: !5) !25 = !DILocation(line: 7, column: 1, scope: !24) !26 = !DILocation(line: 11, column: 14, scope: !12) !27 = !DILocation(line: 11, column: 26, scope: !12) !28 = !DILocation(line: 12, column: 3, scope: !29) !29 = distinct !DILexicalBlock(line: 11, column: 34, file: !32, scope: !12) !30 = !DILocation(line: 13, column: 3, scope: !29) !31 = !DILocation(line: 14, column: 3, scope: !29) !32 = !DIFile(filename: "foo.c", directory: "/tmp") !33 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/constant-pointers.ll000066400000000000000000000055301506646577400257360ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj %t.ll -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Ensure that pointer constants are emitted as unsigned data. Alternatively, ; these could be signless data (dataN). ; Built with Clang from: ; template ; void func() {} ; template void func(); ; CHECK: DW_TAG_subprogram ; CHECK: DW_TAG_template_value_parameter ; CHECK: DW_AT_name {{.*}} "V" ; CHECK: DW_AT_const_value [DW_FORM_udata] (0) ; CHECK: DW_TAG_template_value_parameter ; CHECK: DW_AT_name {{.*}} "F" ; CHECK: DW_AT_const_value [DW_FORM_udata] (0) ; Function Attrs: nounwind uwtable define weak_odr void @_Z4funcILPv0ELPFvvE0ELi42EEvv() #0 !dbg !4 { entry: ret void, !dbg !18 } attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!15, !16} !llvm.ident = !{!17} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "func", linkageName: "_Z4funcILPv0ELPFvvE0ELi42EEvv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, templateParams: !8, retainedNodes: !2) !5 = !DIFile(filename: "constant-pointers.cpp", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = !{!9, !11, !13} !9 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "V", type: !10, value: i8 0) !10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null) !11 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "F", type: !12, value: i8 0) !12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !6) !13 = !DITemplateValueParameter(tag: DW_TAG_template_value_parameter, name: "i", type: !14, value: i32 42) !14 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !15 = !{i32 2, !"Dwarf Version", i32 4} !16 = !{i32 2, !"Debug Info Version", i32 3} !17 = !{!"clang version 3.5.0 "} !18 = !DILocation(line: 3, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/dead-argument-order.ll000066400000000000000000000075051506646577400260760ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Built from the following source with clang -O1 ; struct S { int i; }; ; int function(struct S s, int i) { return s.i + i; } ; Due to the X86_64 ABI, 's' is passed in registers and once optimized, the ; entirety of 's' is never reconstituted, since only the int is required, and ; thus the variable's location is unknown/dead to debug info. ; Future/current work should enable us to describe partial variables, which, in ; this case, happens to be the entire variable. ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "function" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "s" ; CHECK-NOT: DW_TAG ; FIXME: Even though 's' is never reconstituted into a struct, the one member ; variable is still live and used, and so we should be able to describe 's's ; location as the location of that int. ; CHECK-NOT: DW_AT_location ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_location ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "i" %struct.S = type { i32 } ; Function Attrs: nounwind readnone uwtable define i32 @_Z8function1Si(i32 %s.coerce, i32 %i) #0 !dbg !9 { entry: tail call void @llvm.dbg.declare(metadata %struct.S* undef, metadata !14, metadata !DIExpression()), !dbg !20 tail call void @llvm.dbg.value(metadata i32 %i, metadata !15, metadata !DIExpression()), !dbg !20 %add = add nsw i32 %i, %s.coerce, !dbg !20 ret i32 %add, !dbg !20 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!16, !17} !llvm.ident = !{!18} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S") !5 = !{!6} !6 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !4, baseType: !7) !7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = distinct !DISubprogram(name: "function", linkageName: "_Z8function1Si", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 2, file: !1, scope: !10, type: !11, retainedNodes: !13) !10 = !DIFile(filename: "dead-argument-order.cpp", directory: "/tmp/dbginfo") !11 = !DISubroutineType(types: !12) !12 = !{!7, !4, !7} !13 = !{!14, !15} !14 = !DILocalVariable(name: "s", line: 2, arg: 1, scope: !9, file: !10, type: !4) !15 = !DILocalVariable(name: "i", line: 2, arg: 2, scope: !9, file: !10, type: !7) !16 = !{i32 2, !"Dwarf Version", i32 4} !17 = !{i32 2, !"Debug Info Version", i32 3} !18 = !{!"clang version 3.5.0 "} !19 = !{%struct.S* undef} !20 = !DILocation(line: 2, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/debug-info-eis-option.ll000066400000000000000000000100161506646577400263420ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=legacy ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -stop-before=finalize-isel -pre-RA-sched=linearize < %t.ll -experimental-debug-variable-locations=false | FileCheck %s ; RUN: llvm-spirv %t.spv -to-text -o - | FileCheck --check-prefix CHECK-SPIRV %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK-SPIRV: ExtInstImport [[Set:[0-9]+]] "SPIRV.debug" ; CHECK-SPIRV: TypeVoid [[Void:[0-9]+]] ; CHECK-SPIRV: ExtInst [[Void]] {{[0-9]+}} [[Set]] DebugValue source_filename = "linear-dbg-value.ll" ; Function Attrs: nounwind readonly uwtable define i32 @foo(i32* nocapture readonly %a, i32 %N) local_unnamed_addr #0 !dbg !6 { entry: %cmp6 = icmp sgt i32 %N, 0, !dbg !11 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup, !dbg !15 for.body.preheader: ; preds = %entry %wide.trip.count = zext i32 %N to i64 br label %for.body, !dbg !17 for.cond.cleanup.loopexit: ; preds = %for.body br label %for.cond.cleanup, !dbg !19 for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry %x.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.cond.cleanup.loopexit ] ret i32 %x.0.lcssa, !dbg !19 for.body: ; preds = %for.body, %for.body.preheader ; CHECK: ![[X:[0-9]+]] = !DILocalVariable(name: "x", ; CHECK-LABEL: bb.3.for.body: ; CHECK: DBG_VALUE {{.*}} ![[X]], !DIExpression() ; CHECK: DBG_VALUE {{.*}} ![[X]], !DIExpression() %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] %x.07 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ] %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !17 %0 = load i32, i32* %arrayidx, align 4, !dbg !17 %add = add nsw i32 %0, %x.07, !dbg !17 call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !20 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21 call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !20 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count, !dbg !11 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !15 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { nounwind readonly uwtable } attributes #1 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.1 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "foo.c", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{!"clang version 4.0.1 "} !6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8) !7 = !DISubroutineType(types: !2) !8 = !{!9} !9 = !DILocalVariable(name: "x", scope: !6, file: !1, line: 3, type: !10) !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !DILocation(line: 4, scope: !12) !12 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 1) !13 = distinct !DILexicalBlock(scope: !14, file: !1, line: 4, column: 3) !14 = distinct !DILexicalBlock(scope: !6, file: !1, line: 4, column: 3) !15 = !DILocation(line: 4, scope: !16) !16 = !DILexicalBlockFile(scope: !14, file: !1, discriminator: 1) !17 = !DILocation(line: 5, scope: !18) !18 = distinct !DILexicalBlock(scope: !13, file: !1, line: 4, column: 31) !19 = !DILocation(line: 7, scope: !6) !20 = !DILocation(line: 3, scope: !6) !21 = !DILocation(line: 4, scope: !22) !22 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 3) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/def-line.ll000066400000000000000000000112671506646577400237330ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -filetype=obj | llvm-dwarfdump -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Given the following source, ensure that the decl_line/file is correctly ; emitted and omitted on definitions if it mismatches/matches the declaration ; struct foo { ; static void f1() { ; } ; static void f2(); ; static void f3(); ; }; ; void foo::f2() { ; f1(); // just to ensure f1 is emitted ; } ; #line 1 "bar.cpp" ; void foo::f3() { ; } ; Skip the declarations ; CHECK: DW_TAG_subprogram ; CHECK: DW_TAG_subprogram ; CHECK: DW_TAG_subprogram ; CHECK: DW_TAG_subprogram ; CHECK-NOT: {{DW_TAG|NULL|DW_AT_decl_file}} ; CHECK: DW_AT_decl_line {{.*}}7 ; CHECK-NOT: {{DW_TAG|NULL|DW_AT_decl_file}} ; CHECK: DW_AT_specification {{.*}}f2 ; CHECK-NOT: {{DW_TAG|NULL|DW_AT_decl_file}} ; CHECK: DW_TAG_subprogram ; CHECK-NOT: {{DW_TAG|NULL|DW_AT_decl_line|DW_AT_decl_file}} ; CHECK: DW_AT_specification {{.*}}f1 ; CHECK: DW_TAG_subprogram ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_AT_decl_file {{.*}}bar.cpp ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_AT_decl_line {{.*}}1 ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_AT_specification {{.*}}f3 ; Function Attrs: uwtable define void @_ZN3foo2f2Ev() #0 align 2 !dbg !12 { entry: call void @_ZN3foo2f1Ev(), !dbg !19 ret void, !dbg !20 } ; Function Attrs: nounwind uwtable define linkonce_odr void @_ZN3foo2f1Ev() #1 align 2 !dbg !15 { entry: ret void, !dbg !21 } ; Function Attrs: nounwind uwtable define void @_ZN3foo2f3Ev() #1 align 2 !dbg !13 { entry: ret void, !dbg !22 } attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!16, !17} !llvm.ident = !{!18} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 249440) (llvm/trunk 249465)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3) !1 = !DIFile(filename: "def-line.cpp", directory: "/tmp/dbginfo") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !1, line: 1, size: 8, align: 8, elements: !5, identifier: "_ZTS3foo") !5 = !{!6, !9, !10} !6 = !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", scope: !4, file: !1, line: 2, type: !7, isLocal: false, isDefinition: false, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false) !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = !DISubprogram(name: "f2", linkageName: "_ZN3foo2f2Ev", scope: !4, file: !1, line: 4, type: !7, isLocal: false, isDefinition: false, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false) !10 = !DISubprogram(name: "f3", linkageName: "_ZN3foo2f3Ev", scope: !4, file: !1, line: 5, type: !7, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false) !12 = distinct !DISubprogram(name: "f2", linkageName: "_ZN3foo2f2Ev", scope: !4, file: !1, line: 7, type: !7, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !9, retainedNodes: !2) !13 = distinct !DISubprogram(name: "f3", linkageName: "_ZN3foo2f3Ev", scope: !4, file: !14, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !10, retainedNodes: !2) !14 = !DIFile(filename: "bar.cpp", directory: "/tmp/dbginfo") !15 = distinct !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", scope: !4, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !6, retainedNodes: !2) !16 = !{i32 2, !"Dwarf Version", i32 4} !17 = !{i32 2, !"Debug Info Version", i32 3} !18 = !{!"clang version 3.8.0 (trunk 249440) (llvm/trunk 249465)"} !19 = !DILocation(line: 8, column: 3, scope: !12) !20 = !DILocation(line: 9, column: 1, scope: !12) !21 = !DILocation(line: 3, column: 3, scope: !15) !22 = !DILocation(line: 2, column: 1, scope: !13) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/discriminator.ll000066400000000000000000000053571506646577400251220ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -filetype=obj | llvm-dwarfdump -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Given the following source, ensure that the discriminator is emitted for ; the inlined callsite. ;void xyz(); ;static void __attribute__((always_inline)) bar() { xyz(); } ;void foo() { ; bar(); bar(); ;} ;CHECK: DW_TAG_inlined_subroutine ;CHECK-NOT: DW_AT_GNU_discriminator ;CHECK: DW_TAG_inlined_subroutine ;CHECK-NOT: {{DW_TAG|NULL}} ;CHECK: DW_AT_GNU_discriminator{{.*}}0x01 ; Function Attrs: uwtable define void @_Z3foov() #0 !dbg !4 { tail call void @_Z3xyzv(), !dbg !11 tail call void @_Z3xyzv(), !dbg !13 ret void, !dbg !16 } declare void @_Z3xyzv() #1 attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} !llvm.ident = !{!10} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 252497)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "a.cc", directory: "/tmp") !2 = !{} !4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2) !5 = !DISubroutineType(types: !6) !6 = !{null} !7 = distinct !DISubprogram(name: "bar", linkageName: "_ZL3barv", scope: !1, file: !1, line: 2, type: !5, isLocal: true, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2) !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 2, !"Debug Info Version", i32 3} !10 = !{!"clang version 3.8.0 (trunk 252497)"} !11 = !DILocation(line: 2, column: 52, scope: !7, inlinedAt: !12) !12 = distinct !DILocation(line: 4, column: 3, scope: !4) !13 = !DILocation(line: 2, column: 52, scope: !7, inlinedAt: !14) !14 = distinct !DILocation(line: 4, column: 10, scope: !15) !15 = !DILexicalBlockFile(scope: !4, file: !1, discriminator: 1) !16 = !DILocation(line: 5, column: 1, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/dwarf-public-names.ll000066400000000000000000000160741506646577400257310ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -debugger-tune=gdb -filetype=obj -o %t.o < %t.ll ; RUN: llvm-dwarfdump -debug-pubnames %t.o | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ModuleID = 'dwarf-public-names.cpp' ; ; Generated from: ; ; struct C { ; void member_function(); ; static int static_member_function(); ; static int static_member_variable; ; }; ; ; int C::static_member_variable = 0; ; ; void C::member_function() { ; static_member_variable = 0; ; } ; ; int C::static_member_function() { ; return static_member_variable; ; } ; ; C global_variable; ; ; int global_function() { ; return -1; ; } ; ; namespace ns { ; void global_namespace_function() { ; global_variable.member_function(); ; } ; int global_namespace_variable = 1; ; } ; Skip the output to the header of the pubnames section. ; CHECK: debug_pubnames ; CHECK: version = 0x0002 ; Check for each name in the output. ; CHECK-DAG: "ns" ; CHECK-DAG: "C::static_member_function" ; CHECK-DAG: "global_variable" ; CHECK-DAG: "ns::global_namespace_variable" ; CHECK-DAG: "ns::global_namespace_function" ; CHECK-DAG: "global_function" ; CHECK-DAG: "C::static_member_variable" ; CHECK-DAG: "C::member_function" source_filename = "test/DebugInfo/Generic/dwarf-public-names.ll" %struct.C = type { i8 } @_ZN1C22static_member_variableE = global i32 0, align 4, !dbg !0 @global_variable = global %struct.C zeroinitializer, align 1, !dbg !15 @_ZN2ns25global_namespace_variableE = global i32 1, align 4, !dbg !17 ; Function Attrs: nounwind uwtable define void @_ZN1C15member_functionEv(%struct.C* %this) #0 align 2 !dbg !23 { entry: %this.addr = alloca %struct.C*, align 8 store %struct.C* %this, %struct.C** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.C** %this.addr, metadata !24, metadata !26), !dbg !27 %this1 = load %struct.C*, %struct.C** %this.addr store i32 0, i32* @_ZN1C22static_member_variableE, align 4, !dbg !28 ret void, !dbg !29 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 ; Function Attrs: nounwind uwtable define i32 @_ZN1C22static_member_functionEv() #0 align 2 !dbg !30 { entry: %0 = load i32, i32* @_ZN1C22static_member_variableE, align 4, !dbg !31 ret i32 %0, !dbg !31 } ; Function Attrs: nounwind uwtable define i32 @_Z15global_functionv() #0 !dbg !32 { entry: ret i32 -1, !dbg !33 } ; Function Attrs: nounwind uwtable define void @_ZN2ns25global_namespace_functionEv() #0 !dbg !34 { entry: call void @_ZN1C15member_functionEv(%struct.C* @global_variable), !dbg !37 ret void, !dbg !38 } attributes #0 = { nounwind uwtable } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!20} !llvm.module.flags = !{!22} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !2, file: !3, line: 7, type: !6, isLocal: false, isDefinition: true, declaration: !5) !2 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", file: !3, line: 1, size: 8, align: 8, elements: !4) !3 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t") !4 = !{!5, !7, !12} !5 = !DIDerivedType(tag: DW_TAG_member, name: "static_member_variable", scope: !2, file: !3, line: 4, baseType: !6, flags: DIFlagStaticMember) !6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: false, scopeLine: 2, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, retainedNodes: !11) !8 = !DISubroutineType(types: !9) !9 = !{null, !10} !10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !2, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !11 = !{} !12 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !2, file: !3, line: 3, type: !13, isLocal: false, isDefinition: false, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, retainedNodes: !11) !13 = !DISubroutineType(types: !14) !14 = !{!6} !15 = !DIGlobalVariableExpression(var: !16, expr: !DIExpression()) !16 = !DIGlobalVariable(name: "global_variable", scope: null, file: !3, line: 17, type: !2, isLocal: false, isDefinition: true) ; previously: invalid DW_TAG_base_type !17 = !DIGlobalVariableExpression(var: !18, expr: !DIExpression()) !18 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !19, file: !3, line: 27, type: !6, isLocal: false, isDefinition: true) !19 = !DINamespace(name: "ns", scope: null) !20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type !21 = !{!0, !15, !17} !22 = !{i32 1, !"Debug Info Version", i32 3} !23 = distinct !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: null, file: !3, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !7, retainedNodes: !11) !24 = !DILocalVariable(name: "this", arg: 1, scope: !23, file: !3, line: 9, type: !25, flags: DIFlagArtificial | DIFlagObjectPointer) !25 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !2, size: 64, align: 64) !26 = !DIExpression() !27 = !DILocation(line: 9, scope: !23) !28 = !DILocation(line: 10, scope: !23) !29 = !DILocation(line: 11, scope: !23) !30 = distinct !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: null, file: !3, line: 13, type: !13, isLocal: false, isDefinition: true, scopeLine: 13, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !12, retainedNodes: !11) !31 = !DILocation(line: 14, scope: !30) !32 = distinct !DISubprogram(name: "global_function", linkageName: "_Z15global_functionv", scope: !3, file: !3, line: 19, type: !13, isLocal: false, isDefinition: true, scopeLine: 19, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, retainedNodes: !11) !33 = !DILocation(line: 20, scope: !32) !34 = distinct !DISubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", scope: !19, file: !3, line: 24, type: !35, isLocal: false, isDefinition: true, scopeLine: 24, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, retainedNodes: !11) !35 = !DISubroutineType(types: !36) !36 = !{null} !37 = !DILocation(line: 25, scope: !34) !38 = !DILocation(line: 26, scope: !34) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/enum.ll000066400000000000000000000074341506646577400232150ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump -v %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; IR generated from the following code compiled with clang -g: ; enum e1 { I, J = 0xffffffffU, K = 0xf000000000000000ULL } a; ; enum e2 { X }; ; void func() { ; int b = X; ; } ; These values were previously being truncated to -1 and 0 respectively. ; CHECK: debug_info contents ; CHECK: DW_TAG_enumeration_type ; CHECK-NEXT: DW_AT_name{{.*}} = "e1" ; CHECK-NOT: NULL ; CHECK: DW_TAG_enumerator ; CHECK-NOT: NULL ; CHECK: DW_TAG_enumerator ; CHECK-NEXT: DW_AT_name{{.*}} = "J" ; CHECK-NEXT: DW_AT_const_value [DW_FORM_sdata] (4294967295) ; CHECK-NOT: NULL ; CHECK: DW_TAG_enumerator ; CHECK-NEXT: DW_AT_name{{.*}} = "K" ; CHECK-NEXT: DW_AT_const_value [DW_FORM_sdata] (-1152921504606846976) ; Check that we retain enums that aren't referenced by any variables, etc ; CHECK: DW_TAG_enumeration_type ; CHECK-NEXT: DW_AT_name{{.*}} = "e2" ; CHECK-NOT: NULL ; CHECK: DW_TAG_enumerator ; CHECK-NEXT: DW_AT_name{{.*}} = "X" source_filename = "test/DebugInfo/Generic/enum.ll" @a = global i64 0, align 8, !dbg !0 ; Function Attrs: nounwind uwtable define void @_Z4funcv() #0 !dbg !17 { entry: %b = alloca i32, align 4 call void @llvm.dbg.declare(metadata i32* %b, metadata !20, metadata !22), !dbg !23 store i32 0, i32* %b, align 4, !dbg !23 ret void, !dbg !24 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!8} !llvm.module.flags = !{!15, !16} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "a", scope: null, file: !2, line: 1, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "enum.cpp", directory: "/tmp") !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "e1", file: !2, line: 1, size: 64, align: 64, elements: !4) !4 = !{!5, !6, !7} !5 = !DIEnumerator(name: "I", value: 0) !6 = !DIEnumerator(name: "J", value: 4294967295) ; [ DW_TAG_enumerator ] [I :: 0] !7 = !DIEnumerator(name: "K", value: -1152921504606846976) ; [ DW_TAG_enumerator ] [J :: 4294967295] !8 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.4 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !9, retainedTypes: !13, globals: !14, imports: !13) ; [ DW_TAG_enumerator ] [K :: 17293822569102704640] !9 = !{!3, !10} !10 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "e2", file: !2, line: 2, size: 32, align: 32, elements: !11) !11 = !{!12} !12 = !DIEnumerator(name: "X", value: 0) ; [ DW_TAG_enumerator ] [X :: 0] !13 = !{} !14 = !{!0} !15 = !{i32 2, !"Dwarf Version", i32 3} !16 = !{i32 1, !"Debug Info Version", i32 3} !17 = distinct !DISubprogram(name: "func", linkageName: "_Z4funcv", scope: !2, file: !2, line: 3, type: !18, isLocal: false, isDefinition: true, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !8, retainedNodes: !13) !18 = !DISubroutineType(types: !19) !19 = !{null} !20 = !DILocalVariable(name: "b", scope: !17, file: !2, line: 4, type: !21) !21 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !22 = !DIExpression() !23 = !DILocation(line: 4, scope: !17) !24 = !DILocation(line: 5, scope: !17) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/func-using-decl.ll000066400000000000000000000057171506646577400252360ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Generated from the following source: ; extern void func_test1(int, int); ; namespace my_ns ; { ; using ::func_test1; ; } ; int main () ; { ; return 0; ; } ; Ensure forward routine declarations with an associated using declaration ; are resolved properly (temporary node would trigger an assert). ; CHECK: DW_TAG_namespace ; CHECK: DW_AT_name {{.*}}"my_ns" ; CHECK: DW_TAG_imported_declaration ; CHECK: NULL ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_name {{.*}}"func_test1" ; CHECK: NULL source_filename = "test/DebugInfo/Generic/func-using-decl.ll" ; Function Attrs: noinline norecurse nounwind optnone uwtable define dso_local i32 @main() #0 !dbg !14 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval, align 4 ret i32 0, !dbg !17 } attributes #0 = { noinline norecurse nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11, !12} !llvm.ident = !{!13} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, imports: !3, nameTableKind: None) !1 = !DIFile(filename: "func-using-decl.cpp", directory: "/tmp") !2 = !{} !3 = !{!4} !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !5, entity: !6, file: !1, line: 4) !5 = !DINamespace(name: "my_ns", scope: null) !6 = !DISubprogram(name: "func_test1", linkageName: "_Z10func_test1ii", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false) !7 = !DISubroutineType(types: !8) !8 = !{null, !9, !9} !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !10 = !{i32 2, !"Dwarf Version", i32 4} !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{i32 1, !"wchar_size", i32 4} !13 = !{!"clang version 8.0.0"} !14 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 6, type: !15, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !15 = !DISubroutineType(types: !16) !16 = !{!9} !17 = !DILocation(line: 8, column: 3, scope: !14) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/function-ptr-templ-null.ll000066400000000000000000000123731506646577400267660ustar00rootroot00000000000000;; compiled from (null added manually): ;; template ;; int foo() { ;; int result = Func(); ;; return result; ;; }; ;; ;; long get() { return 42; } ;; ;; void boo() { ;; int val = foo(); ;; } ; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck < %t.ll %s --check-prefix=CHECK-LLVM ; RUN: llc -mtriple=x86_64-linux -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s --check-prefix=CHECK-DWARF ; CHECK-LLVM: ![[#]] = !DITemplateValueParameter(name: "Func", type: ![[#Type:]], value: ptr null) ; CHECK-LLVM: ![[#Type]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[#]], size: 64) ; CHECK-DWARF: DW_TAG_subprogram ; CHECK-DWARF: DW_AT_name{{.*}}"foo<&get>" ; CHECK-DWARF: DW_TAG_template_value_parameter ; CHECK-DWARF: DW_AT_type {{.*}} "int (*)()" ; CHECK-DWARF: DW_AT_name {{.*}} "Func" ; ModuleID = '/app/example.cpp' source_filename = "/app/example.cpp" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" $_Z3fooIXadL_Z3getvEEEiv = comdat any ; Function Attrs: mustprogress noinline nounwind optnone uwtable define dso_local noundef i32 @_Z3getv() #0 !dbg !10 { ret i32 42, !dbg !16 } ; Function Attrs: mustprogress noinline optnone uwtable define dso_local void @_Z3boov() #1 !dbg !17 { %1 = alloca i32, align 4 call void @llvm.dbg.declare(metadata ptr %1, metadata !20, metadata !DIExpression()), !dbg !21 %2 = call noundef i32 @_Z3fooIXadL_Z3getvEEEiv(), !dbg !22 store i32 %2, ptr %1, align 4, !dbg !21 ret void, !dbg !23 } ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 ; Function Attrs: mustprogress noinline nounwind optnone uwtable define linkonce_odr dso_local noundef i32 @_Z3fooIXadL_Z3getvEEEiv() #0 comdat !dbg !24 { %1 = alloca i32, align 4 call void @llvm.dbg.declare(metadata ptr %1, metadata !28, metadata !DIExpression()), !dbg !29 %2 = call noundef i32 @_Z3getv(), !dbg !30 store i32 %2, ptr %1, align 4, !dbg !29 %3 = load i32, ptr %1, align 4, !dbg !31 ret i32 %3, !dbg !32 } attributes #0 = { mustprogress noinline nounwind optnone uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } attributes #1 = { mustprogress noinline optnone uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} !llvm.ident = !{!9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git ef38880ce03bc1f1fb3606c5a629151f3d0e975e)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "/app/example.cpp", directory: "/app") !2 = !{i32 7, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 8, !"PIC Level", i32 2} !6 = !{i32 7, !"PIE Level", i32 2} !7 = !{i32 7, !"uwtable", i32 2} !8 = !{i32 7, !"frame-pointer", i32 2} !9 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git ef38880ce03bc1f1fb3606c5a629151f3d0e975e)"} !10 = distinct !DISubprogram(name: "get", linkageName: "_Z3getv", scope: !11, file: !11, line: 7, type: !12, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !11 = !DIFile(filename: "example.cpp", directory: "/app") !12 = !DISubroutineType(types: !13) !13 = !{!14} !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !{} !16 = !DILocation(line: 7, column: 13, scope: !10) !17 = distinct !DISubprogram(name: "boo", linkageName: "_Z3boov", scope: !11, file: !11, line: 9, type: !18, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !18 = !DISubroutineType(types: !19) !19 = !{null} !20 = !DILocalVariable(name: "val", scope: !17, file: !11, line: 10, type: !14) !21 = !DILocation(line: 10, column: 9, scope: !17) !22 = !DILocation(line: 10, column: 15, scope: !17) !23 = !DILocation(line: 11, column: 1, scope: !17) !24 = distinct !DISubprogram(name: "foo<&get>", linkageName: "_Z3fooIXadL_Z3getvEEEiv", scope: !11, file: !11, line: 2, type: !12, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, templateParams: !25, retainedNodes: !15) !25 = !{!26} !26 = !DITemplateValueParameter(name: "Func", type: !27, value: null) !27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64) !28 = !DILocalVariable(name: "result", scope: !24, file: !11, line: 3, type: !14) !29 = !DILocation(line: 3, column: 9, scope: !24) !30 = !DILocation(line: 3, column: 18, scope: !24) !31 = !DILocation(line: 4, column: 12, scope: !24) !32 = !DILocation(line: 4, column: 5, scope: !24) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/function-ptr-templ.ll000066400000000000000000000123571506646577400260200ustar00rootroot00000000000000;; compiled from: ;; template ;; int foo() { ;; int result = Func(); ;; return result; ;; }; ;; ;; long get() { return 42; } ;; ;; void boo() { ;; int val = foo(); ;; } ; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck < %t.ll %s --check-prefix=CHECK-LLVM ; RUN: llc -mtriple=x86_64-linux -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s --check-prefix=CHECK-DWARF ; CHECK-LLVM: ![[#]] = !DITemplateValueParameter(name: "Func", type: ![[#Type:]], value: ptr @_Z3getv) ; CHECK-LLVM: ![[#Type]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[#]], size: 64) ; CHECK-DWARF: DW_TAG_subprogram ; CHECK-DWARF: DW_AT_name{{.*}}"foo<&get>" ; CHECK-DWARF: DW_TAG_template_value_parameter ; CHECK-DWARF: DW_AT_type {{.*}} "int (*)()" ; CHECK-DWARF: DW_AT_name {{.*}} "Func" ; ModuleID = '/app/example.cpp' source_filename = "/app/example.cpp" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" $_Z3fooIXadL_Z3getvEEEiv = comdat any ; Function Attrs: mustprogress noinline nounwind optnone uwtable define dso_local noundef i32 @_Z3getv() #0 !dbg !10 { ret i32 42, !dbg !16 } ; Function Attrs: mustprogress noinline optnone uwtable define dso_local void @_Z3boov() #1 !dbg !17 { %1 = alloca i32, align 4 call void @llvm.dbg.declare(metadata ptr %1, metadata !20, metadata !DIExpression()), !dbg !21 %2 = call noundef i32 @_Z3fooIXadL_Z3getvEEEiv(), !dbg !22 store i32 %2, ptr %1, align 4, !dbg !21 ret void, !dbg !23 } ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 ; Function Attrs: mustprogress noinline nounwind optnone uwtable define linkonce_odr dso_local noundef i32 @_Z3fooIXadL_Z3getvEEEiv() #0 comdat !dbg !24 { %1 = alloca i32, align 4 call void @llvm.dbg.declare(metadata ptr %1, metadata !28, metadata !DIExpression()), !dbg !29 %2 = call noundef i32 @_Z3getv(), !dbg !30 store i32 %2, ptr %1, align 4, !dbg !29 %3 = load i32, ptr %1, align 4, !dbg !31 ret i32 %3, !dbg !32 } attributes #0 = { mustprogress noinline nounwind optnone uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } attributes #1 = { mustprogress noinline optnone uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} !llvm.ident = !{!9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git ef38880ce03bc1f1fb3606c5a629151f3d0e975e)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "/app/example.cpp", directory: "/app") !2 = !{i32 7, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 8, !"PIC Level", i32 2} !6 = !{i32 7, !"PIE Level", i32 2} !7 = !{i32 7, !"uwtable", i32 2} !8 = !{i32 7, !"frame-pointer", i32 2} !9 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git ef38880ce03bc1f1fb3606c5a629151f3d0e975e)"} !10 = distinct !DISubprogram(name: "get", linkageName: "_Z3getv", scope: !11, file: !11, line: 7, type: !12, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !11 = !DIFile(filename: "example.cpp", directory: "/app") !12 = !DISubroutineType(types: !13) !13 = !{!14} !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !{} !16 = !DILocation(line: 7, column: 13, scope: !10) !17 = distinct !DISubprogram(name: "boo", linkageName: "_Z3boov", scope: !11, file: !11, line: 9, type: !18, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !15) !18 = !DISubroutineType(types: !19) !19 = !{null} !20 = !DILocalVariable(name: "val", scope: !17, file: !11, line: 10, type: !14) !21 = !DILocation(line: 10, column: 9, scope: !17) !22 = !DILocation(line: 10, column: 15, scope: !17) !23 = !DILocation(line: 11, column: 1, scope: !17) !24 = distinct !DISubprogram(name: "foo<&get>", linkageName: "_Z3fooIXadL_Z3getvEEEiv", scope: !11, file: !11, line: 2, type: !12, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, templateParams: !25, retainedNodes: !15) !25 = !{!26} !26 = !DITemplateValueParameter(name: "Func", type: !27, value: ptr @_Z3getv) !27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64) !28 = !DILocalVariable(name: "result", scope: !24, file: !11, line: 3, type: !14) !29 = !DILocation(line: 3, column: 9, scope: !24) !30 = !DILocation(line: 3, column: 18, scope: !24) !31 = !DILocation(line: 4, column: 12, scope: !24) !32 = !DILocation(line: 4, column: 5, scope: !24) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/global.ll000066400000000000000000000040621506646577400235030ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump %t | FileCheck %s ; Also test that the null streamer doesn't crash with debug info. ; RUN: llc -mtriple=%triple -O0 -filetype=null < %t.ll target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; generated from the following source compiled to bitcode with clang -g -O1 ; static int i; ; int main() { ; (void)&i; ; } ; CHECK: debug_info contents ; CHECK: DW_TAG_variable source_filename = "test/DebugInfo/Generic/global.ll" ; Function Attrs: nounwind readnone uwtable define i32 @main() #0 !dbg !9 { entry: ret i32 0, !dbg !12 } attributes #0 = { nounwind readnone uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!7, !8} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.4 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !3, imports: !2) !1 = !DIFile(filename: "global.cpp", directory: "/tmp") !2 = !{} !3 = !{!4} !4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression()) !5 = !DIGlobalVariable(name: "i", linkageName: "_ZL1i", scope: null, file: !1, line: 1, type: !6, isLocal: true, isDefinition: true) !6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !{i32 2, !"Dwarf Version", i32 3} !8 = !{i32 1, !"Debug Info Version", i32 3} !9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !10, isLocal: false, isDefinition: true, scopeLine: 2, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2) !10 = !DISubroutineType(types: !11) !11 = !{!6} !12 = !DILocation(line: 4, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/gmlt_profiling.ll000066400000000000000000000031771506646577400252650ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %S/gmlt_profiling.ll | llvm-dwarfdump -v - | FileCheck %S/gmlt_profiling.ll target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: .debug_info ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "f1" ; With debug-info-for-profiling attribute, we need to emit decl_file and ; decl_line of the subprogram. ; CHECK-NEXT: DW_AT_decl_file ; CHECK-NEXT: DW_AT_decl_line ; Function Attrs: nounwind uwtable define void @_Z2f1v() !dbg !4 { entry: ret void, !dbg !13 } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11} !llvm.ident = !{!12} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2, debugInfoForProfiling: true) !1 = !DIFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "f1", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "gmlt.cpp", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !2) !10 = !{i32 2, !"Dwarf Version", i32 4} !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{!"clang version 3.6.0 "} !13 = !DILocation(line: 1, column: 12, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/imported-name-inlined.ll000066400000000000000000000057541506646577400264350ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Generated from the following source: ; namespace ns { ; void f(); ; } ; inline __attribute__((always_inline)) void f1() { ; using ns::f; ; f(); ; } ; void f2() { f1(); } ; Ensure that top level imported declarations don't produce an extra degenerate ; concrete subprogram definition. ; FIXME: imported entities should only be emitted to the abstract origin if one is present ; CHECK: DW_TAG_compile_unit ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_name {{.*}} "f1" ; CHECK: DW_TAG_imported_declaration ; CHECK: NULL ; CHECK: DW_TAG_namespace ; CHECK: DW_TAG_subprogram ; CHECK: NULL ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_name {{.*}} "f2" ; CHECK: DW_TAG_inlined_subroutine ; CHECK: DW_TAG_imported_declaration ; CHECK: NULL ; CHECK: NULL ; CHECK: NULL ; Function Attrs: noinline define void @_Z2f2v() noinline !dbg !14 { entry: call void @_ZN2ns1fEv(), !dbg !15 ret void, !dbg !17 } declare void @_ZN2ns1fEv() !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11, !12} !llvm.ident = !{!13} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 6.0.0 (trunk 309061) (llvm/trunk 309076)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, imports: !3) !1 = !DIFile(filename: "imported-name-inlined.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch") !2 = !{} !3 = !{!4} !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !5, entity: !8, file: !1, line: 5) !5 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 4, type: !6, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = !DISubprogram(name: "f", linkageName: "_ZN2ns1fEv", scope: !9, file: !1, line: 2, type: !6, isLocal: false, isDefinition: false, flags: DIFlagPrototyped, isOptimized: false) !9 = !DINamespace(name: "ns", scope: null) !10 = !{i32 2, !"Dwarf Version", i32 4} !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{i32 1, !"wchar_size", i32 4} !13 = !{!"clang version 6.0.0 (trunk 309061) (llvm/trunk 309076)"} !14 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 8, type: !6, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !15 = !DILocation(line: 6, column: 3, scope: !5, inlinedAt: !16) !16 = distinct !DILocation(line: 8, column: 13, scope: !14) !17 = !DILocation(line: 8, column: 19, scope: !14) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll000066400000000000000000000077701506646577400276720ustar00rootroot00000000000000; REQUIRES: object-emission ; This test is failing for powerpc64, because a location list for the ; variable 'c' is not generated at all. Temporary marking this test as XFAIL ; for powerpc, until PR21881 is fixed. ; XFAIL: powerpc64 ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O2 -dwarf-version 2 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF23 ; RUN: llc -mtriple=%triple -O2 -dwarf-version 3 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF23 ; RUN: llc -mtriple=%triple -O2 -dwarf-version 4 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck %s --check-prefix=DWARF4 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; This is a test for PR21176. ; DW_OP_const doesn't describe a constant value, but a value at a constant address. ; The proper way to describe a constant value is DW_OP_constu , DW_OP_stack_value. ; For values < 32 we emit the canonical DW_OP_lit. ; Generated with clang -S -emit-llvm -g -O2 test.cpp ; extern int func(); ; ; int main() ; { ; volatile int c = 13; ; c = func(); ; return c; ; } ; CHECK: DW_TAG_variable ; CHECK: DW_AT_location ; CHECK-NOT: DW_AT ; DWARF23: DW_OP_lit13{{$}} ; DWARF4: DW_OP_lit13, DW_OP_stack_value{{$}} ; Function Attrs: uwtable define i32 @main() #0 !dbg !4 { entry: %c = alloca i32, align 4 tail call void @llvm.dbg.value(metadata i32 13, metadata !10, metadata !16), !dbg !17 store volatile i32 13, i32* %c, align 4, !dbg !18 %call = tail call i32 @_Z4funcv(), !dbg !19 tail call void @llvm.dbg.value(metadata i32 %call, metadata !10, metadata !16), !dbg !17 store volatile i32 %call, i32* %c, align 4, !dbg !19 tail call void @llvm.dbg.value(metadata i32* %c, metadata !10, metadata !21), !dbg !17 %c.0.c.0. = load volatile i32, i32* %c, align 4, !dbg !20 ret i32 %c.0.c.0., !dbg !20 } declare i32 @_Z4funcv() #1 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #2 attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!12, !13} !llvm.ident = !{!14} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 223522)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !9) !5 = !DIFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{!10} !10 = !DILocalVariable(name: "c", line: 5, scope: !4, file: !5, type: !11) !11 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !8) !12 = !{i32 2, !"Dwarf Version", i32 2} !13 = !{i32 2, !"Debug Info Version", i32 3} !14 = !{!"clang version 3.6.0 (trunk 223522)"} !15 = !{i32 13} !16 = !DIExpression() !17 = !DILocation(line: 5, column: 16, scope: !4) !18 = !DILocation(line: 5, column: 3, scope: !4) !19 = !DILocation(line: 6, column: 7, scope: !4) !20 = !DILocation(line: 7, column: 3, scope: !4) !21 = !DIExpression(DW_OP_deref) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/inline-scopes.ll000066400000000000000000000141641506646577400250170ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; bool f(); ; inline __attribute__((always_inline)) int f1() { ; if (bool b = f()) ; return 1; ; return 2; ; } ; ; inline __attribute__((always_inline)) int f2() { ; # 2 "y.cc" ; if (bool b = f()) ; return 3; ; return 4; ; } ; ; int main() { ; f1(); ; f2(); ; } ; Ensure that lexical_blocks within inlined_subroutines are preserved/emitted. ; CHECK: DW_TAG_inlined_subroutine ; CHECK-NOT: DW_TAG ; CHECK-NOT: NULL ; CHECK: DW_TAG_lexical_block ; CHECK-NOT: DW_TAG ; CHECK-NOT: NULL ; CHECK: DW_TAG_variable ; Ensure that file changes don't interfere with creating inlined subroutines. ; (see the line directive inside 'f2' in thesource) ; CHECK: DW_TAG_inlined_subroutine ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin ; Function Attrs: uwtable define i32 @main() #0 !dbg !4 { entry: %retval.i2 = alloca i32, align 4 %b.i3 = alloca i8, align 1 %retval.i = alloca i32, align 4 %b.i = alloca i8, align 1 call void @llvm.dbg.declare(metadata i8* %b.i, metadata !16, metadata !DIExpression()), !dbg !19 %call.i = call zeroext i1 @_Z1fv(), !dbg !19 %frombool.i = zext i1 %call.i to i8, !dbg !19 store i8 %frombool.i, i8* %b.i, align 1, !dbg !19 %0 = load i8, i8* %b.i, align 1, !dbg !19 %tobool.i = trunc i8 %0 to i1, !dbg !19 br i1 %tobool.i, label %if.then.i, label %if.end.i, !dbg !19 if.then.i: ; preds = %entry store i32 1, i32* %retval.i, !dbg !21 br label %_Z2f1v.exit, !dbg !21 if.end.i: ; preds = %entry store i32 2, i32* %retval.i, !dbg !22 br label %_Z2f1v.exit, !dbg !22 _Z2f1v.exit: ; preds = %if.then.i, %if.end.i %1 = load i32, i32* %retval.i, !dbg !23 call void @llvm.dbg.declare(metadata i8* %b.i3, metadata !24, metadata !DIExpression()), !dbg !27 %call.i4 = call zeroext i1 @_Z1fv(), !dbg !27 %frombool.i5 = zext i1 %call.i4 to i8, !dbg !27 store i8 %frombool.i5, i8* %b.i3, align 1, !dbg !27 %2 = load i8, i8* %b.i3, align 1, !dbg !27 %tobool.i6 = trunc i8 %2 to i1, !dbg !27 br i1 %tobool.i6, label %if.then.i7, label %if.end.i8, !dbg !27 if.then.i7: ; preds = %_Z2f1v.exit store i32 3, i32* %retval.i2, !dbg !29 br label %_Z2f2v.exit, !dbg !29 if.end.i8: ; preds = %_Z2f1v.exit store i32 4, i32* %retval.i2, !dbg !30 br label %_Z2f2v.exit, !dbg !30 _Z2f2v.exit: ; preds = %if.then.i7, %if.end.i8 %3 = load i32, i32* %retval.i2, !dbg !31 ret i32 0, !dbg !32 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 declare zeroext i1 @_Z1fv() #2 attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!13, !14} !llvm.ident = !{!15} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !5, scope: !6, type: !7, retainedNodes: !2) !5 = !DIFile(filename: "y.cc", directory: "/tmp/dbginfo") !6 = !DIFile(filename: "y.cc", directory: "/tmp/dbginfo") !7 = !DISubroutineType(types: !8) !8 = !{!9} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !10 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 8, file: !1, scope: !11, type: !7, retainedNodes: !2) !11 = !DIFile(filename: "inline-scopes.cpp", directory: "/tmp/dbginfo") !12 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !11, type: !7, retainedNodes: !2) !13 = !{i32 2, !"Dwarf Version", i32 4} !14 = !{i32 1, !"Debug Info Version", i32 3} !15 = !{!"clang version 3.5.0 "} !16 = !DILocalVariable(name: "b", line: 3, scope: !17, file: !11, type: !18) !17 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !12) !18 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean) !19 = !DILocation(line: 3, scope: !17, inlinedAt: !20) !20 = !DILocation(line: 8, scope: !4) !21 = !DILocation(line: 4, scope: !17, inlinedAt: !20) !22 = !DILocation(line: 5, scope: !12, inlinedAt: !20) !23 = !DILocation(line: 6, scope: !12, inlinedAt: !20) !24 = !DILocalVariable(name: "b", line: 2, scope: !25, file: !6, type: !18) !25 = distinct !DILexicalBlock(line: 2, column: 0, file: !5, scope: !26) !26 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !10) !27 = !DILocation(line: 2, scope: !25, inlinedAt: !28) !28 = !DILocation(line: 9, scope: !4) !29 = !DILocation(line: 3, scope: !25, inlinedAt: !28) !30 = !DILocation(line: 4, scope: !26, inlinedAt: !28) !31 = !DILocation(line: 5, scope: !26, inlinedAt: !28) !32 = !DILocation(line: 10, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/inlined-arguments.ll000066400000000000000000000074171506646577400256770ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; IR generated from clang -O -g with the following source ; ; void f1(int x, int y); ; void f3(int line); ; void f2() { ; f1(1, 2); ; } ; void f1(int x, int y) { ; f3(y); ; } ; CHECK: DW_AT_name{{.*}}"f1" ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name{{.*}}"x" ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name{{.*}}"y" ; Function Attrs: uwtable define void @_Z2f2v() #0 !dbg !4 { tail call void @llvm.dbg.value(metadata i32 undef, metadata !16, metadata !DIExpression()), !dbg !18 tail call void @llvm.dbg.value(metadata i32 2, metadata !20, metadata !DIExpression()), !dbg !18 tail call void @_Z2f3i(i32 2), !dbg !21 ret void, !dbg !22 } ; Function Attrs: uwtable define void @_Z2f1ii(i32 %x, i32 %y) #0 !dbg !8 { tail call void @llvm.dbg.value(metadata i32 %x, metadata !13, metadata !DIExpression()), !dbg !23 tail call void @llvm.dbg.value(metadata i32 %y, metadata !14, metadata !DIExpression()), !dbg !23 tail call void @_Z2f3i(i32 %y), !dbg !24 ret void, !dbg !25 } declare void @_Z2f3i(i32) #1 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #2 attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!26} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch") !2 = !{} !4 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "exp.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch") !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1ii", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 6, file: !1, scope: !5, type: !9, retainedNodes: !12) !9 = !DISubroutineType(types: !10) !10 = !{null, !11, !11} !11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !12 = !{!13, !14} !13 = !DILocalVariable(name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11) !14 = !DILocalVariable(name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11) !15 = !{i32 undef} !16 = !DILocalVariable(name: "x", line: 6, arg: 1, scope: !8, file: !5, type: !11) !17 = !DILocation(line: 4, scope: !4) !18 = !DILocation(line: 6, scope: !8, inlinedAt: !17) !19 = !{i32 2} !20 = !DILocalVariable(name: "y", line: 6, arg: 2, scope: !8, file: !5, type: !11) !21 = !DILocation(line: 7, scope: !8, inlinedAt: !17) !22 = !DILocation(line: 5, scope: !4) !23 = !DILocation(line: 6, scope: !8) !24 = !DILocation(line: 7, scope: !8) !25 = !DILocation(line: 8, scope: !8) !26 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/inlined-locations.ll000066400000000000000000000121221506646577400256520ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefixes=CHECK,CHECK-200 ; Check that the "inlinedAt" attribute of a DILocation references another ; DILocation that is marked as distinct. Note that the checks for distinct ; DILocations do not include the column number as SPIR-V does not allow for ; representing this info (except for NonSemantic.Shader.DebugInfo.200). For ; this specification we check that the column number is preserved during ; translation. ; Built with clang -O -g from the source: ; bool f(); ; inline __attribute__((always_inline)) int f1() { ; if (bool b = f()) ; return 1; ; return 0; ; } ; ; inline __attribute__((always_inline)) int f2() { ; if (int i = f1()) ; return 3; ; return 4; ; } ; ; int main() { ; f2(); ; } source_filename = "test/DebugInfo/Generic/inlined-locations.ll" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64-unknown-unknown" ; Function Attrs: norecurse define dso_local i32 @main() local_unnamed_addr #0 !dbg !7 { %1 = tail call spir_func zeroext i1 @_Z1fv(), !dbg !11 call void @llvm.dbg.value(metadata i8 undef, metadata !16, metadata !DIExpression()), !dbg !24 ret i32 0, !dbg !25 } declare dso_local spir_func zeroext i1 @_Z1fv() local_unnamed_addr #1 ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.value(metadata, metadata, metadata) #2 attributes #0 = { norecurse "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "inlined-locations.cpp", directory: "/tmp/dbginfo") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{!"clang version 9.0.0 "} !7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !8, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !8 = !DISubroutineType(types: !9) !9 = !{!10} !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !DILocation(line: 3, column: 16, scope: !12, inlinedAt: !18) ; CHECK: !{{.*}} = !DILocation(line: 3, column: 16, scope: !{{.*}}, inlinedAt: ![[loc1:[0-9]+]]) !12 = distinct !DILexicalBlock(scope: !13, file: !1, line: 3, column: 12) !13 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 2, type: !14, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !15) !14 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !9) !15 = !{!16} !16 = !DILocalVariable(name: "b", scope: !12, file: !1, line: 3, type: !17) !17 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean) !18 = distinct !DILocation(line: 9, column: 15, scope: !19, inlinedAt: !23) ; CHECK-DEFAULT: ![[loc1]] = distinct !DILocation(line: 9, scope: !{{.*}}, inlinedAt: ![[loc2:[0-9]+]]) ; CHECK-200: ![[loc1]] = distinct !DILocation(line: 9, column: 15, scope: !{{.*}}, inlinedAt: ![[loc2:[0-9]+]]) !19 = distinct !DILexicalBlock(scope: !20, file: !1, line: 9, column: 11) !20 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 8, type: !14, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !21) !21 = !{!22} !22 = !DILocalVariable(name: "i", scope: !19, file: !1, line: 9, type: !10) !23 = distinct !DILocation(line: 15, column: 3, scope: !7) ; CHECK-DEFAULT: ![[loc2]] = distinct !DILocation(line: 15, scope: !{{.*}}) ; CHECK-200: ![[loc2]] = distinct !DILocation(line: 15, column: 3, scope: !{{.*}}) !24 = !DILocation(line: 3, column: 12, scope: !12, inlinedAt: !18) ; CHECK: !{{.*}} = !DILocation(line: 3, column: 12, scope: !{{.*}}, inlinedAt: ![[loc1]]) !25 = !DILocation(line: 16, column: 1, scope: !7) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/inlined-vars.ll000066400000000000000000000060631506646577400246410ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 < %t.ll | FileCheck %s -check-prefix ARGUMENT ; RUN: llc -mtriple=%triple -O0 < %t.ll | FileCheck %s -check-prefix VARIABLE target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; PR 13202 define i32 @main() uwtable !dbg !5 { entry: tail call void @llvm.dbg.value(metadata i32 0, metadata !18, metadata !DIExpression()), !dbg !21 tail call void @llvm.dbg.value(metadata i32 0, metadata !22, metadata !DIExpression()), !dbg !23 tail call void @smth(i32 0), !dbg !24 tail call void @smth(i32 0), !dbg !25 ret i32 0, !dbg !19 } declare void @smth(i32) declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!27} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 159419)", isOptimized: true, emissionKind: FullDebug, file: !26, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !{i32 0} !2 = !{} !5 = distinct !DISubprogram(name: "main", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !26, scope: !6, type: !7, retainedNodes: !2) !6 = !DIFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202") !7 = !DISubroutineType(types: !8) !8 = !{!9} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !10 = distinct !DISubprogram(name: "f", linkageName: "_ZL1fi", line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !26, scope: !6, type: !11, retainedNodes: !13) !11 = !DISubroutineType(types: !12) !12 = !{!9, !9} !13 = !{!15, !16} !15 = !DILocalVariable(name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9) ; Two DW_TAG_formal_parameter: one abstract and one inlined. ; ARGUMENT: {{.*Abbrev.*DW_TAG_formal_parameter}} ; ARGUMENT: {{.*Abbrev.*DW_TAG_formal_parameter}} ; ARGUMENT-NOT: {{.*Abbrev.*DW_TAG_formal_parameter}} !16 = !DILocalVariable(name: "local", line: 4, scope: !10, file: !6, type: !9) ; Two DW_TAG_variable: one abstract and one inlined. ; VARIABLE: {{.*Abbrev.*DW_TAG_variable}} ; VARIABLE: {{.*Abbrev.*DW_TAG_variable}} ; VARIABLE-NOT: {{.*Abbrev.*DW_TAG_variable}} !18 = !DILocalVariable(name: "argument", line: 3, arg: 1, scope: !10, file: !6, type: !9) !19 = !DILocation(line: 11, column: 10, scope: !5) !21 = !DILocation(line: 3, column: 25, scope: !10, inlinedAt: !19) !22 = !DILocalVariable(name: "local", line: 4, scope: !10, file: !6, type: !9) !23 = !DILocation(line: 4, column: 16, scope: !10, inlinedAt: !19) !24 = !DILocation(line: 5, column: 3, scope: !10, inlinedAt: !19) !25 = !DILocation(line: 6, column: 3, scope: !10, inlinedAt: !19) !26 = !DIFile(filename: "inline-bug.cc", directory: "/tmp/dbginfo/pr13202") !27 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/linear-dbg-value.ll000066400000000000000000000077651506646577400253760ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -stop-before=finalize-isel -pre-RA-sched=linearize < %t.ll -experimental-debug-variable-locations=false | FileCheck %s ; RUN: llvm-spirv %t.spv -to-text -o - | FileCheck --check-prefix CHECK-SPIRV %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK-SPIRV: ExtInstImport [[Set:[0-9]+]] "OpenCL.DebugInfo.100" ; CHECK-SPIRV: TypeVoid [[Void:[0-9]+]] ; CHECK-SPIRV: ExtInst [[Void]] {{[0-9]+}} [[Set]] DebugValue source_filename = "linear-dbg-value.ll" ; Function Attrs: nounwind readonly uwtable define i32 @foo(i32* nocapture readonly %a, i32 %N) local_unnamed_addr #0 !dbg !6 { entry: %cmp6 = icmp sgt i32 %N, 0, !dbg !11 br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup, !dbg !15 for.body.preheader: ; preds = %entry %wide.trip.count = zext i32 %N to i64 br label %for.body, !dbg !17 for.cond.cleanup.loopexit: ; preds = %for.body br label %for.cond.cleanup, !dbg !19 for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry %x.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.cond.cleanup.loopexit ] ret i32 %x.0.lcssa, !dbg !19 for.body: ; preds = %for.body, %for.body.preheader ; CHECK: ![[X:[0-9]+]] = !DILocalVariable(name: "x", ; CHECK-LABEL: bb.3.for.body: ; CHECK: DBG_VALUE {{.*}} ![[X]], !DIExpression() ; CHECK: DBG_VALUE {{.*}} ![[X]], !DIExpression() %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ] %x.07 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ] %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !17 %0 = load i32, i32* %arrayidx, align 4, !dbg !17 %add = add nsw i32 %0, %x.07, !dbg !17 call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !20 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21 call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !20 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count, !dbg !11 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !15 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { nounwind readonly uwtable } attributes #1 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.1 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "foo.c", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{!"clang version 4.0.1 "} !6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8) !7 = !DISubroutineType(types: !2) !8 = !{!9} !9 = !DILocalVariable(name: "x", scope: !6, file: !1, line: 3, type: !10) !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !DILocation(line: 4, scope: !12) !12 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 1) !13 = distinct !DILexicalBlock(scope: !14, file: !1, line: 4, column: 3) !14 = distinct !DILexicalBlock(scope: !6, file: !1, line: 4, column: 3) !15 = !DILocation(line: 4, scope: !16) !16 = !DILexicalBlockFile(scope: !14, file: !1, discriminator: 1) !17 = !DILocation(line: 5, scope: !18) !18 = distinct !DILexicalBlock(scope: !13, file: !1, line: 4, column: 31) !19 = !DILocation(line: 7, scope: !6) !20 = !DILocation(line: 3, scope: !6) !21 = !DILocation(line: 4, scope: !22) !22 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 3) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/linkage-name-abstract.ll000066400000000000000000000132701506646577400263750ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj -dwarf-linkage-names=Abstract < %t.ll | llvm-dwarfdump -v -debug-info - > %t ; RUN: FileCheck %s -check-prefix=ONENAME < %t ; RUN: FileCheck %s -check-prefix=REF < %t ; Verify tuning for SCE gets us Abstract only. ; RUN: llc -mtriple=%triple -O0 -filetype=obj -debugger-tune=sce < %t.ll | llvm-dwarfdump -v -debug-info - > %t ; RUN: FileCheck %s -check-prefix=ONENAME < %t ; RUN: FileCheck %s -check-prefix=REF < %t ; REQUIRES: object-emission ; Verify that the only linkage-name present is the abstract origin of the ; inlined subprogram. ; IR generated from clang -O0 with: ; void f1(); ; __attribute__((always_inline)) void f2() { ; f1(); ; } ; void f3() { ; f2(); ; } ; ; struct F4 { ; __attribute__((always_inline)) void f5(); ; }; ; void F4::f5() { ; f1(); ; } ; void f6() { ; F4::f5(); ; } ; Show that the only linkage names are for the inlined functions, ; because those are the ones with an abstract origin. ; ONENAME-NOT: {{DW_AT(_MIPS)?_linkage_name}} ; ONENAME: {{DW_AT(_MIPS)?_linkage_name}} {{.*}} "_Z2f2v" ; ONENAME-NOT: {{DW_AT(_MIPS)?_linkage_name}} ; ONENAME: {{DW_AT(_MIPS)?_linkage_name}} {{.*}} "_ZN2F42f5Ev" ; ONENAME-NOT: {{DW_AT(_MIPS)?_linkage_name}} ; For f2() we see the definition pointing to an abstract origin DIE, ; which in turn is where the linkage_name is; and then there's ; an inlined_subroutine pointing back to the abstract origin. ; The order of these DIEs is not important of course, just the links. ; REF: DW_TAG_subprogram ; REF-NOT: {{DW_TAG|NULL}} ; REF: DW_AT_abstract_origin {{.*}} {[[F2:0x.*]]} "_Z2f2v" ; REF: [[F2]]: DW_TAG_subprogram ; REF-NEXT: linkage_name {{.*}} "_Z2f2v" ; REF: DW_TAG_inlined_subroutine ; REF-NOT: {{DW_TAG|NULL}} ; REF: DW_AT_abstract_origin {{.*}} {[[F2]]} ; For F4::f5(), first we see the in-class declaration, ; then the definition, abstract origin, and the inlined_subroutine. ; REF: DW_TAG_structure_type ; REF-NEXT: DW_AT_name {{.*}} "F4" ; REF-NOT: {{DW_TAG|NULL}} ; REF: [[F5_DECL:0x.*]]: DW_TAG_subprogram ; REF-NEXT: DW_AT_name {{.*}} "f5" ; REF: DW_TAG_subprogram ; REF-NOT: {{DW_TAG|NULL}} ; REF: DW_AT_abstract_origin {{.*}} {[[F5_ABS:0x.*]]} "_ZN2F42f5Ev" ; REF: [[F5_ABS]]: DW_TAG_subprogram ; REF-NOT: {{DW_TAG|NULL}} ; REF: linkage_name {{.*}} "_ZN2F42f5Ev" ; REF-NEXT: DW_AT_specification {{.*}} {[[F5_DECL]]} ; REF: DW_TAG_inlined_subroutine ; REF-NOT: {{DW_TAG|NULL}} ; REF: DW_AT_abstract_origin {{.*}} {[[F5_ABS]]} target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Function Attrs: alwaysinline uwtable define void @_Z2f2v() #0 !dbg !6 { entry: call void @_Z2f1v(), !dbg !9 ret void, !dbg !10 } declare void @_Z2f1v() ; Function Attrs: uwtable define void @_Z2f3v() !dbg !11 { entry: call void @_Z2f1v(), !dbg !12 ret void, !dbg !14 } ; Function Attrs: alwaysinline uwtable define void @_ZN2F42f5Ev() #0 align 2 !dbg !15 { entry: call void @_Z2f1v(), !dbg !19 ret void, !dbg !20 } ; Function Attrs: uwtable define void @_Z2f6v() !dbg !21 { entry: call void @_Z2f1v(), !dbg !22 ret void, !dbg !24 } attributes #0 = { alwaysinline } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0 (trunk 288231)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "linkage-name-abstract-static.cpp", directory: "/home/probinson/projects/scratch") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{!"clang version 4.0.0 (trunk 288231)"} !6 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = !DILocation(line: 3, column: 3, scope: !6) !10 = !DILocation(line: 4, column: 1, scope: !6) !11 = distinct !DISubprogram(name: "f3", linkageName: "_Z2f3v", scope: !1, file: !1, line: 5, type: !7, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !12 = !DILocation(line: 3, column: 3, scope: !6, inlinedAt: !13) !13 = distinct !DILocation(line: 6, column: 3, scope: !11) !14 = !DILocation(line: 7, column: 1, scope: !11) !15 = distinct !DISubprogram(name: "f5", linkageName: "_ZN2F42f5Ev", scope: !16, file: !1, line: 12, type: !7, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: false, unit: !0, declaration: !18, retainedNodes: !2) !16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "F4", file: !1, line: 9, size: 8, elements: !17, identifier: "_ZTS2F4") !17 = !{!18} !18 = !DISubprogram(name: "f5", linkageName: "_ZN2F42f5Ev", scope: !16, file: !1, line: 10, type: !7, isLocal: false, isDefinition: false, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: false) !19 = !DILocation(line: 13, column: 3, scope: !15) !20 = !DILocation(line: 14, column: 1, scope: !15) !21 = distinct !DISubprogram(name: "f6", linkageName: "_Z2f6v", scope: !1, file: !1, line: 15, type: !7, isLocal: false, isDefinition: true, scopeLine: 15, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !22 = !DILocation(line: 13, column: 3, scope: !15, inlinedAt: !23) !23 = distinct !DILocation(line: 16, column: 3, scope: !21) !24 = !DILocation(line: 17, column: 1, scope: !21) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/member-order.ll000066400000000000000000000064611506646577400246300ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=obj -O0 < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; generated by clang from: ; struct foo { ; void f1(); ; void f2(); ; }; ; ; void foo::f1() { ; } ; CHECK: DW_TAG_structure_type ; CHECK-NEXT: DW_AT_name {{.*}} "foo" ; CHECK-NOT: NULL ; CHECK: DW_TAG_subprogram ; CHECK-NOT: NULL ; CHECK: DW_AT_name {{.*}} "f1" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: NULL ; CHECK: DW_AT_name {{.*}} "f2" %struct.foo = type { i8 } ; Function Attrs: nounwind uwtable define void @_ZN3foo2f1Ev(%struct.foo* %this) #0 align 2 !dbg !14 { entry: %this.addr = alloca %struct.foo*, align 8 store %struct.foo* %this, %struct.foo** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.foo** %this.addr, metadata !16, metadata !DIExpression()), !dbg !18 %this1 = load %struct.foo*, %struct.foo** %this.addr ret void, !dbg !19 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!15, !20} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "member-order.cpp", directory: "/tmp/dbginfo") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS3foo") !5 = !{!6, !11} !6 = !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !4, type: !7) !7 = !DISubroutineType(types: !8) !8 = !{null, !9} !9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4) !10 = !{i32 786468} !11 = !DISubprogram(name: "f2", linkageName: "_ZN3foo2f2Ev", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !4, type: !7) !12 = !{i32 786468} !14 = distinct !DISubprogram(name: "f1", linkageName: "_ZN3foo2f1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: null, type: !7, declaration: !6, retainedNodes: !2) !15 = !{i32 2, !"Dwarf Version", i32 4} !16 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !14, type: !17) !17 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4) !18 = !DILocation(line: 0, scope: !14) !19 = !DILocation(line: 7, scope: !14) !20 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/missing-abstract-variable.ll000066400000000000000000000167411506646577400273070ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Build from the following source with clang -O2. ; The important details are that 'x's abstract definition is first built during ; the definition of 'b', where the parameter to 'x' is constant and so 'x's 's' ; variable is optimized away. No abstract definition DIE for 's' is constructed. ; Then, during 'a' emission, the abstract DbgVariable for 's' is created, but ; the abstract DIE isn't (since the abstract definition for 'b' is already ; built). This results in 's' inlined in 'a' being emitted with its name, line, ; file there, rather than referencing an abstract definition. ; extern int t; ; ; void f(int); ; ; inline void x(bool b) { ; if (b) { ; int s = t; ; f(s); ; } ; f(0); ; } ; ; void b() { ; x(false); ; } ; ; void a(bool u) { ; x(u); ; } ; CHECK: [[X_DECL:.*]]: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "x" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "b" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_lexical_block ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "s" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "b" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_inlined_subroutine ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} {[[X_DECL]]} ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "b" ; Notice 'x's local variable 's' is missing. Not necessarily a bug here, ; since it's been optimized entirely away and it should be described in ; abstract subprogram. ; CHECK-NOT: DW_TAG ; CHECK: NULL ; CHECK-NOT: DW_TAG ; CHECK: NULL ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "a" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_inlined_subroutine ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} {[[X_DECL]]} ; CHECK-NOT: {{DW_TAG|NULL}} ; FIXME: This formal parameter goes missing at least at -O2 (& on ; mips/powerpc), maybe before that. Perhaps SelectionDAG is to blame (and ; fastisel succeeds). ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "b" ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_lexical_block ; CHECK-NOT: {{DW_TAG|NULL}} ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "s" @t = external global i32 ; Function Attrs: uwtable define void @_Z1bv() #0 !dbg !4 { entry: tail call void @llvm.dbg.value(metadata i1 false, metadata !25, metadata !DIExpression()), !dbg !27 tail call void @_Z1fi(i32 0), !dbg !28 ret void, !dbg !29 } ; Function Attrs: uwtable define void @_Z1ab(i1 zeroext %u) #0 !dbg !8 { entry: tail call void @llvm.dbg.value(metadata i1 %u, metadata !13, metadata !DIExpression()), !dbg !30 tail call void @llvm.dbg.value(metadata i1 %u, metadata !31, metadata !DIExpression()), !dbg !33 br i1 %u, label %if.then.i, label %_Z1xb.exit, !dbg !34 if.then.i: ; preds = %entry %0 = load i32, i32* @t, align 4, !dbg !35, !tbaa !36 tail call void @llvm.dbg.value(metadata i32 %0, metadata !40, metadata !DIExpression()), !dbg !35 tail call void @_Z1fi(i32 %0), !dbg !41 br label %_Z1xb.exit, !dbg !42 _Z1xb.exit: ; preds = %entry, %if.then.i tail call void @_Z1fi(i32 0), !dbg !43 ret void, !dbg !44 } declare void @_Z1fi(i32) #1 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #2 attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!21, !22} !llvm.ident = !{!23} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "b", linkageName: "_Z1bv", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 13, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "missing-abstract-variables.cc", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = distinct !DISubprogram(name: "a", linkageName: "_Z1ab", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 17, file: !1, scope: !5, type: !9, retainedNodes: !12) !9 = !DISubroutineType(types: !10) !10 = !{null, !11} !11 = !DIBasicType(tag: DW_TAG_base_type, name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean) !12 = !{!13} !13 = !DILocalVariable(name: "u", line: 17, arg: 1, scope: !8, file: !5, type: !11) !14 = distinct !DISubprogram(name: "x", linkageName: "_Z1xb", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 5, file: !1, scope: !5, type: !9, retainedNodes: !15) !15 = !{!16, !17} !16 = !DILocalVariable(name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11) !17 = !DILocalVariable(name: "s", line: 7, scope: !18, file: !5, type: !20) !18 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !19) !19 = distinct !DILexicalBlock(line: 6, column: 0, file: !1, scope: !14) !20 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !21 = !{i32 2, !"Dwarf Version", i32 4} !22 = !{i32 2, !"Debug Info Version", i32 3} !23 = !{!"clang version 3.5.0 "} !24 = !{i1 false} !25 = !DILocalVariable(name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11) !26 = !DILocation(line: 14, scope: !4) !27 = !DILocation(line: 5, scope: !14, inlinedAt: !26) !28 = !DILocation(line: 10, scope: !14, inlinedAt: !26) !29 = !DILocation(line: 15, scope: !4) !30 = !DILocation(line: 17, scope: !8) !31 = !DILocalVariable(name: "b", line: 5, arg: 1, scope: !14, file: !5, type: !11) !32 = !DILocation(line: 18, scope: !8) !33 = !DILocation(line: 5, scope: !14, inlinedAt: !32) !34 = !DILocation(line: 6, scope: !19, inlinedAt: !32) !35 = !DILocation(line: 7, scope: !18, inlinedAt: !32) !36 = !{!37, !37, i64 0} !37 = !{!"int", !38, i64 0} !38 = !{!"omnipotent char", !39, i64 0} !39 = !{!"Simple C/C++ TBAA"} !40 = !DILocalVariable(name: "s", line: 7, scope: !18, file: !5, type: !20) !41 = !DILocation(line: 8, scope: !18, inlinedAt: !32) !42 = !DILocation(line: 9, scope: !18, inlinedAt: !32) !43 = !DILocation(line: 10, scope: !14, inlinedAt: !32) !44 = !DILocation(line: 19, scope: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/multiline.ll000066400000000000000000000065511506646577400242520ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=asm -asm-verbose=0 -O0 < %t.ll | FileCheck %s ; RUN: llc -mtriple=%triple -filetype=obj -O0 < %t.ll | llvm-dwarfdump -debug-line - | FileCheck %s --check-prefix=INT target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Check that the assembly output properly handles is_stmt changes. And since ; we're testing anyway, check the integrated assembler too. ; Generated with clang from multiline.c: ; void f1(); ; void f2() { ; f1(); f1(); f1(); ; f1(); f1(); f1(); ; } ; CHECK: .loc 1 2 0{{$}} ; CHECK-NOT: .loc{{ }} ; CHECK: .loc 1 3 3 prologue_end{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 3 9 is_stmt 0{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 3 15{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 4 3 is_stmt 1{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 4 9 is_stmt 0{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 4 15{{$}} ; CHECK-NOT: .loc ; CHECK: .loc 1 5 1 is_stmt 1{{$}} ; INT: {{^}}Address ; INT: ----- ; INT-NEXT: 2 0 1 0 0 is_stmt{{$}} ; INT-NEXT: 3 3 1 0 0 is_stmt prologue_end{{$}} ; INT-NEXT: 3 9 1 0 0 {{$}} ; INT-NEXT: 3 15 1 0 0 {{$}} ; INT-NEXT: 4 3 1 0 0 is_stmt{{$}} ; INT-NEXT: 4 9 1 0 0 {{$}} ; INT-NEXT: 4 15 1 0 0 {{$}} ; INT-NEXT: 5 1 1 0 0 is_stmt{{$}} ; Function Attrs: nounwind uwtable define void @f2() #0 !dbg !4 { entry: call void (...) @f1(), !dbg !11 call void (...) @f1(), !dbg !12 call void (...) @f1(), !dbg !13 call void (...) @f1(), !dbg !14 call void (...) @f1(), !dbg !15 call void (...) @f1(), !dbg !16 ret void, !dbg !17 } declare void @f1(...) #1 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} !llvm.ident = !{!10} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 225000) (llvm/trunk 224999)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "multiline.c", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "f2", line: 2, isLocal: false, isDefinition: true, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "multiline.c", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 2, !"Debug Info Version", i32 3} !10 = !{!"clang version 3.6.0 (trunk 225000) (llvm/trunk 224999)"} !11 = !DILocation(line: 3, column: 3, scope: !4) !12 = !DILocation(line: 3, column: 9, scope: !4) !13 = !DILocation(line: 3, column: 15, scope: !4) !14 = !DILocation(line: 4, column: 3, scope: !4) !15 = !DILocation(line: 4, column: 9, scope: !4) !16 = !DILocation(line: 4, column: 15, scope: !4) !17 = !DILocation(line: 5, column: 1, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/namespace_function_definition.ll000066400000000000000000000037311506646577400303160ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj -dwarf-linkage-names=All < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Generated from clang with the following source: ; namespace ns { ; void func() { ; } ; } ; CHECK: DW_TAG_namespace ; CHECK-NEXT: DW_AT_name {{.*}} "ns" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_low_pc ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_linkage_name {{.*}} "_ZN2ns4funcEv" ; CHECK: NULL ; CHECK: NULL ; Function Attrs: nounwind uwtable define void @_ZN2ns4funcEv() #0 !dbg !4 { entry: ret void, !dbg !11 } attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!8, !9} !llvm.ident = !{!10} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "namespace_function_definition.cpp", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "func", linkageName: "_ZN2ns4funcEv", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DINamespace(name: "ns", scope: null) !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 1, !"Debug Info Version", i32 3} !10 = !{!"clang version 3.5.0 "} !11 = !DILocation(line: 3, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/namespace_inline_function_definition.ll000066400000000000000000000105741506646577400316570ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj -dwarf-linkage-names=All < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Generate from clang with the following source. Note that the definition of ; the inline function follows its use to workaround another bug that should be ; fixed soon. ; namespace ns { ; int func(int i); ; } ; extern int x; ; int main() { return ns::func(x); } ; int __attribute__((always_inline)) ns::func(int i) { return i * 2; } ; CHECK: DW_TAG_namespace ; CHECK-NEXT: DW_AT_name {{.*}} "ns" ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_linkage_name {{.*}} "_ZN2ns4funcEi" ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_formal_parameter ; CHECK: NULL ; CHECK-NOT: NULL ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_abstract_origin {{.*}} "_ZN2ns4funcEi" ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_formal_parameter ; CHECK: DW_AT_abstract_origin {{.*}} "i" ; CHECK: NULL ; CHECK: NULL ; CHECK: NULL @x = external global i32 ; Function Attrs: uwtable define i32 @main() #0 !dbg !4 { entry: %i.addr.i = alloca i32, align 4 %retval = alloca i32, align 4 store i32 0, i32* %retval %0 = load i32, i32* @x, align 4, !dbg !16 store i32 %0, i32* %i.addr.i, align 4 call void @llvm.dbg.declare(metadata i32* %i.addr.i, metadata !117, metadata !DIExpression()), !dbg !18 %1 = load i32, i32* %i.addr.i, align 4, !dbg !18 %mul.i = mul nsw i32 %1, 2, !dbg !18 ret i32 %mul.i, !dbg !16 } ; Function Attrs: alwaysinline nounwind uwtable define i32 @_ZN2ns4funcEi(i32 %i) #1 !dbg !9 { entry: %i.addr = alloca i32, align 4 store i32 %i, i32* %i.addr, align 4 call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !17, metadata !DIExpression()), !dbg !19 %0 = load i32, i32* %i.addr, align 4, !dbg !19 %mul = mul nsw i32 %0, 2, !dbg !19 ret i32 %mul, !dbg !19 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!13, !14} !llvm.ident = !{!15} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 5, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "namespace_inline_function_definition.cpp", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = distinct !DISubprogram(name: "func", linkageName: "_ZN2ns4funcEi", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: !10, type: !11, retainedNodes: !2) !10 = !DINamespace(name: "ns", scope: null) !11 = !DISubroutineType(types: !12) !12 = !{!8, !8} !13 = !{i32 2, !"Dwarf Version", i32 4} !14 = !{i32 2, !"Debug Info Version", i32 3} !15 = !{!"clang version 3.5.0 "} !16 = !DILocation(line: 5, scope: !4) !17 = !DILocalVariable(name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8) !117 = !DILocalVariable(name: "i", line: 6, arg: 1, scope: !9, file: !5, type: !8) !18 = !DILocation(line: 6, scope: !9, inlinedAt: !16) !19 = !DILocation(line: 6, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/noscopes.ll000066400000000000000000000027311506646577400240750ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Just because there are no scopes/locations on any instructions in the ; function doesn't mean we can't describe the address range of the function. ; Check that we do that ; CHECK: DW_TAG_subprogram ; CHECK-NOT: TAG ; CHECK: DW_AT_low_pc ; Function Attrs: nounwind uwtable define void @f() #0 !dbg !6 { entry: ret void } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 289692) (llvm/trunk 289697)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "noscopes.c", directory: "/tmp/dbginfo") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{!"clang version 4.0.0 (trunk 289692) (llvm/trunk 289697)"} !6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2) !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = !DILocation(line: 2, column: 1, scope: !6) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/pass-by-value.ll000066400000000000000000000057701506646577400247420ustar00rootroot00000000000000; REQUIRES: object-emission ; ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; ; RUN: llc -mtriple=%triple -O0 -filetype=obj %t.ll -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ; // S is not trivially copyable. ; struct S { ; ~S() {} ; }; ; ; // T is a POD. ; struct T { ; ~T() = default; ; }; ; ; S s; ; T t; ; ; CHECK: DW_TAG_structure_type ; CHECK-NEXT: DW_AT_calling_convention {{.*}} (DW_CC_pass_by_reference) ; CHECK-NEXT: DW_AT_name {{.*}} "S" ; ; CHECK: DW_TAG_structure_type ; CHECK-NEXT: DW_AT_calling_convention {{.*}} (DW_CC_pass_by_value) ; CHECK-NEXT: DW_AT_name {{.*}} "T" %struct.S = type { i8 } %struct.T = type { i8 } @s = global %struct.S zeroinitializer, align 1, !dbg !0 @__dso_handle = external hidden global i8 @t = global %struct.T zeroinitializer, align 1, !dbg !6 !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!20, !21, !22, !23} !llvm.ident = !{!24} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "s", scope: !2, file: !3, line: 9, type: !14, isLocal: false, isDefinition: true) !2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 7.0.0 (trunk 321763) (llvm/trunk 321758)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5) !3 = !DIFile(filename: "pass.cpp", directory: "/") !4 = !{} !5 = !{!0, !6} !6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression()) !7 = distinct !DIGlobalVariable(name: "t", scope: !2, file: !3, line: 10, type: !8, isLocal: false, isDefinition: true) !8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "T", file: !3, line: 5, size: 8, elements: !9, identifier: "_ZTS1T", flags: DIFlagTypePassByValue) !9 = !{!10} !10 = !DISubprogram(name: "~T", scope: !8, file: !3, line: 6, type: !11, isLocal: false, isDefinition: false, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: false) !11 = !DISubroutineType(types: !12) !12 = !{null, !13} !13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !3, line: 1, size: 8, elements: !15, identifier: "_ZTS1S", flags: DIFlagTypePassByReference) !15 = !{!16} !16 = !DISubprogram(name: "~S", scope: !14, file: !3, line: 2, type: !17, isLocal: false, isDefinition: false, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false) !17 = !DISubroutineType(types: !18) !18 = !{null, !19} !19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !20 = !{i32 2, !"Dwarf Version", i32 4} !21 = !{i32 2, !"Debug Info Version", i32 3} !22 = !{i32 1, !"wchar_size", i32 4} !23 = !{i32 7, !"PIC Level", i32 2} !24 = !{!"clang version 7.0.0 (trunk 321763) (llvm/trunk 321758)"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/ptrsize.ll000066400000000000000000000042611506646577400237440ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump -v %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Check that pointers and references get emitted without size information in ; DWARF, even if they are so specified in the IR ; CHECK: 0x[[O1:[0-9a-f]+]]: DW_TAG_pointer_type ; CHECK-NEXT: DW_AT_type [DW_FORM_ref4] ; CHECK-NOT: DW_AT_byte_size ; CHECK: 0x[[O2:[0-9a-f]+]]: DW_TAG_ ; CHECK: 0x[[O3:[0-9a-f]+]]: DW_TAG_reference_type ; CHECK-NEXT: DW_AT_type [DW_FORM_ref4] ; CHECK-NOT: DW_AT_byte_size define i32 @foo() !dbg !4 { entry: ret i32 0, !dbg !13 } define i32 @bar() !dbg !5 { entry: ret i32 0, !dbg !16 } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!11, !12} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "dwarf-test.c", directory: "test") !2 = !{} !4 = distinct !DISubprogram(name: "foo", scope: !0, file: !1, line: 6, type: !6, isLocal: false, isDefinition: true, scopeLine: 6, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !5 = distinct !DISubprogram(name: "bar", scope: !0, file: !1, line: 6, type: !15, isLocal: false, isDefinition: true, scopeLine: 6, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !6 = !DISubroutineType(types: !7) !7 = !{!9} !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !DIDerivedType(tag: DW_TAG_pointer_type, scope: !0, baseType: !8, size: 64, align: 64) !10 = !DIDerivedType(tag: DW_TAG_reference_type, scope: !0, baseType: !8, size: 64, align: 64) !11 = !{i32 2, !"Dwarf Version", i32 3} !12 = !{i32 1, !"Debug Info Version", i32 3} !13 = !DILocation(line: 7, scope: !4) !14 = !{!10} !15 = !DISubroutineType(types: !14) !16 = !DILocation(line: 7, scope: !5) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/restrict.ll000066400000000000000000000053531506646577400241060ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -dwarf-version=2 -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck --check-prefix=CHECK --check-prefix=V2 %s ; RUN: llc -mtriple=%triple -dwarf-version=3 -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck --check-prefix=CHECK --check-prefix=V3 %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: DW_AT_name {{.*}} "dst" ; V2: DW_AT_type {{.*}} {[[PTR:0x.*]]} ; V3: DW_AT_type {{.*}} {[[RESTRICT:0x.*]]} ; V3: [[RESTRICT]]: {{.*}}DW_TAG_restrict_type ; V3-NEXT: DW_AT_type {{.*}} {[[PTR:0x.*]]} ; CHECK: [[PTR]]: {{.*}}DW_TAG_pointer_type ; CHECK-NOT: DW_AT_type ; Generated with clang from: ; void foo(void* __restrict__ dst) { ; } ; Function Attrs: nounwind uwtable define void @_Z3fooPv(i8* noalias %dst) #0 !dbg !4 { entry: %dst.addr = alloca i8*, align 8 store i8* %dst, i8** %dst.addr, align 8 call void @llvm.dbg.declare(metadata i8** %dst.addr, metadata !13, metadata !DIExpression()), !dbg !14 ret void, !dbg !15 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11} !llvm.ident = !{!12} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "restrict.c", directory: "/tmp/dbginfo") !2 = !{} !4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooPv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "restrict.c", directory: "/tmp/dbginfo") !6 = !DISubroutineType(types: !7) !7 = !{null, !8} !8 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !9) !9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null) !10 = !{i32 2, !"Dwarf Version", i32 4} !11 = !{i32 1, !"Debug Info Version", i32 3} !12 = !{!"clang version 3.5.0 "} !13 = !DILocalVariable(name: "dst", line: 1, arg: 1, scope: !4, file: !5, type: !8) !14 = !DILocation(line: 1, scope: !4) !15 = !DILocation(line: 2, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/templ-func-decl.ll000066400000000000000000000077371506646577400252360ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-linux -O0 -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s ; IR generated with `clang -Xclang -debug-info-kind=limited -emit-llvm -S` from the following code: ; class A { ; public: ; template static int foo() { return T; } ; }; ; ; int main() { ; A::template foo<42>(); ; } ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_name{{.*}}"foo<42>" ; CHECK: DW_TAG_template_value_parameter ; CHECK: DW_AT_type {{.*}} "int" ; CHECK: DW_AT_name {{.*}} "T" ; CHECK: DW_AT_const_value {{.*}} (42) ; ModuleID = 'templ-func-decl.cpp' source_filename = "templ-func-decl.cpp" target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" $_ZN1A3fooILi42EEEiv = comdat any ; Function Attrs: noinline norecurse optnone uwtable define dso_local i32 @main() #0 !dbg !6 { entry: %call = call i32 @_ZN1A3fooILi42EEEiv(), !dbg !10 ret i32 0, !dbg !11 } ; Function Attrs: noinline nounwind optnone uwtable define linkonce_odr dso_local i32 @_ZN1A3fooILi42EEEiv() #1 comdat align 2 !dbg !12 { entry: ret i32 42, !dbg !17 } attributes #0 = { noinline norecurse optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "templ-func-decl.cpp", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{!"clang version 8.0.0 "} !6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 6, type: !7, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !7 = !DISubroutineType(types: !8) !8 = !{!9} !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !10 = !DILocation(line: 7, column: 3, scope: !6) !11 = !DILocation(line: 8, column: 1, scope: !6) !12 = distinct !DISubprogram(name: "foo<42>", linkageName: "_ZN1A3fooILi42EEEiv", scope: !13, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, templateParams: !15, declaration: !14, retainedNodes: !2) !13 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS1A") !14 = !DISubprogram(name: "foo<42>", linkageName: "_ZN1A3fooILi42EEEiv", scope: !13, file: !1, line: 3, type: !7, isLocal: false, isDefinition: false, scopeLine: 3, flags: DIFlagPublic | DIFlagPrototyped | DIFlagStaticMember, isOptimized: false, templateParams: !15) !15 = !{!16} !16 = !DITemplateValueParameter(name: "T", type: !9, value: i32 42) !17 = !DILocation(line: 3, column: 39, scope: !12) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/template-recursive-void.ll000066400000000000000000000073341506646577400270270ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump -v %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; This was pulled from clang's debug-info-template-recursive.cpp test. ; class base { }; ; template class foo : public base { ; void operator=(const foo r) { } ; }; ; class bar : public foo { }; ; bar filters; ; CHECK: DW_TAG_template_type_parameter [{{.*}}] ; CHECK-NEXT: DW_AT_name{{.*}}"T" ; CHECK-NOT: DW_AT_type ; CHECK: {{DW_TAG|NULL}} source_filename = "test/DebugInfo/Generic/template-recursive-void.ll" %class.bar = type { i8 } @filters = global %class.bar zeroinitializer, align 1, !dbg !0 !llvm.dbg.cu = !{!29} !llvm.module.flags = !{!32, !33} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "filters", scope: null, file: !2, line: 10, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "debug-info-template-recursive.cpp", directory: "/usr/local/google/home/echristo/tmp") !3 = !DICompositeType(tag: DW_TAG_class_type, name: "bar", file: !2, line: 9, size: 8, align: 8, elements: !4) !4 = !{!5, !25} !5 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !3, baseType: !6) !6 = !DICompositeType(tag: DW_TAG_class_type, name: "foo", file: !2, line: 5, size: 8, align: 8, elements: !7, templateParams: !23) !7 = !{!8, !15, !20} !8 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !6, baseType: !9) !9 = !DICompositeType(tag: DW_TAG_class_type, name: "base", file: !2, line: 3, size: 8, align: 8, elements: !10) !10 = !{!11} !11 = !DISubprogram(name: "base", scope: !9, file: !2, line: 3, type: !12, isLocal: false, isDefinition: false, scopeLine: 3, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false) !12 = !DISubroutineType(types: !13) !13 = !{null, !14} !14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !15 = !DISubprogram(name: "operator=", linkageName: "_ZN3fooIvEaSES0_", scope: !6, file: !2, line: 6, type: !16, isLocal: false, isDefinition: false, scopeLine: 6, virtualIndex: 6, flags: DIFlagPrivate | DIFlagPrototyped, isOptimized: false) !16 = !DISubroutineType(types: !17) !17 = !{null, !18, !19} !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !19 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !6) !20 = !DISubprogram(name: "foo", scope: !6, file: !2, line: 5, type: !21, isLocal: false, isDefinition: false, scopeLine: 5, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false) !21 = !DISubroutineType(types: !22) !22 = !{null, !18} !23 = !{!24} !24 = !DITemplateTypeParameter(name: "T", type: null) !25 = !DISubprogram(name: "bar", scope: !3, file: !2, line: 9, type: !26, isLocal: false, isDefinition: false, scopeLine: 9, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false) !26 = !DISubroutineType(types: !27) !27 = !{null, !28} !28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !29 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.4 (trunk 187958) (llvm/trunk 187964)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !30, retainedTypes: !30, globals: !31, imports: !30) !30 = !{} !31 = !{!0} !32 = !{i32 2, !"Dwarf Version", i32 3} !33 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/tu-member-opaque.spt000066400000000000000000000037551506646577400256370ustar00rootroot00000000000000; RUN: cat %s > %t.spvasm ; RUN: llvm-spirv %t.spvasm --to-binary -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s ; SPIR-V generated via: ; clang -cc1 input.cl -triple spir -disable-llvm-passes -debug-info-kind=standalone -emit-llvm-bc -o - | ./bin/llvm-spirv -spirv-mem2reg -o input.spv; llvm-spirv input.spv --to-text -o - ; where input.cl is: ; struct Foo { ; int e; ; }; ; void foo() { ; struct Foo f; ; } ; ; The resulting SPIR-V was then tweaked to have the DebugTypeComposite use a ; DebugInfoNone (%7) instead of the real size (%18). The resulting debug info ; then contains a DICompositeType with no size information. ; CHECK: {{![0-9]*}} = !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: {{![0-9]*}}, line: 1, elements: {{![0-9]*}}) 119734787 65536 393230 8 0 2 Capability Addresses 2 Capability Linkage 2 Capability Kernel 5 ExtInstImport 1 "OpenCL.std" 5 ExtInstImport 2 "SPIRV.debug" 3 MemoryModel 1 2 6 String 9 "/tmp/" 3 String 14 "Foo" 6 String 15 "/tmp/input.cl" 3 String 19 "e" 3 String 20 "int" 3 String 25 "foo" 3 String 26 "" 3 String 28 "f" 3 Source 3 100000 3 Name 5 "foo" 4 Name 6 "entry" 5 Name 30 "struct.Foo" 5 Decorate 5 LinkageAttributes "foo" Export 4 TypeInt 17 32 0 4 Constant 17 18 32 4 Constant 17 22 0 2 TypeVoid 3 3 TypeFunction 4 3 3 TypeStruct 30 17 4 TypePointer 31 7 30 3 Undef 31 32 5 ExtInst 3 7 2 DebugInfoNone 7 ExtInst 3 10 2 DebugSource 9 7 9 ExtInst 3 11 2 DebugCompilationUnit 65536 0 10 12 7 ExtInst 3 12 2 DebugTypeFunction 0 7 7 ExtInst 3 16 2 DebugSource 15 7 8 ExtInst 3 21 2 DebugTypeBasic 20 18 4 14 ExtInst 3 23 2 DebugTypeMember 19 21 16 2 0 13 22 18 0 15 ExtInst 3 13 2 DebugTypeComposite 14 1 16 1 0 11 7 7 0 23 16 ExtInst 3 27 2 DebugFunction 25 12 16 4 0 11 26 8328 4 5 7 12 ExtInst 3 29 2 DebugLocalVariable 28 13 16 5 0 27 0 5 ExtInst 3 33 2 DebugExpression 5 Function 3 5 0 4 2 Label 6 6 ExtInst 3 34 2 DebugScope 27 4 Line 15 5 0 8 ExtInst 3 8 2 DebugDeclare 29 32 33 4 Line 15 6 0 1 Return 1 FunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/tu-member-pointer.ll000066400000000000000000000032341506646577400256160ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=obj -O0 < %t.ll > %t ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: DW_TAG_ptr_to_member_type ; CHECK-NEXT: DW_AT_type [DW_FORM_ref4] (cu + {{.*}} => {[[TYPE:0x[0-9a-f]+]]} ; CHECK: [[TYPE]]: DW_TAG_base_type ; IR generated from clang -g with the following source: ; struct Foo { ; int e; ; }; ; int Foo:*x = 0; source_filename = "test/DebugInfo/Generic/tu-member-pointer.ll" @x = global i64 -1, align 8, !dbg !0 !llvm.dbg.cu = !{!6} !llvm.module.flags = !{!10, !11} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "x", scope: null, file: !2, line: 4, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "foo.cpp", directory: ".") !3 = !DIDerivedType(tag: DW_TAG_ptr_to_member_type, baseType: !4, extraData: !5) !4 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !5 = !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: !2, line: 1, flags: DIFlagFwdDecl, identifier: "_ZTS3Foo") !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.4", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !7, retainedTypes: !8, globals: !9, imports: !7) !7 = !{} !8 = !{!5} !9 = !{!0} !10 = !{i32 2, !"Dwarf Version", i32 2} !11 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/two-cus-from-same-file.ll000066400000000000000000000074531506646577400264540ustar00rootroot00000000000000; For http://llvm.org/bugs/show_bug.cgi?id=12942 ; There are two CUs coming from /tmp/foo.c in this module. Make sure it doesn't ; blow llc up and produces something reasonable. ; ; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple %t.ll -o %t -filetype=obj -O0 ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ModuleID = 'test.bc' @str = private unnamed_addr constant [4 x i8] c"FOO\00" @str1 = private unnamed_addr constant [6 x i8] c"Main!\00" define void @foo() nounwind !dbg !5 { entry: %puts = tail call i32 @puts(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @str, i32 0, i32 0)), !dbg !23 ret void, !dbg !25 } declare i32 @puts(i8* nocapture) nounwind define i32 @main(i32 %argc, i8** nocapture %argv) nounwind !dbg !12 { entry: tail call void @llvm.dbg.value(metadata i32 %argc, metadata !21, metadata !DIExpression()), !dbg !26 ; Avoid talking about the pointer size in debug info because that's target dependent tail call void @llvm.dbg.value(metadata i8** %argv, metadata !22, metadata !DIExpression(DW_OP_deref, DW_OP_deref)), !dbg !27 %puts = tail call i32 @puts(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @str1, i32 0, i32 0)), !dbg !28 tail call void @foo() nounwind, !dbg !30 ret i32 0, !dbg !31 } declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0, !9} !llvm.module.flags = !{!33} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: FullDebug, file: !32, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !1 = !{} !5 = distinct !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 5, file: !32, scope: !6, type: !7, retainedNodes: !1) !6 = !DIFile(filename: "foo.c", directory: "/tmp") !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.2 (trunk 156513)", isOptimized: true, emissionKind: FullDebug, file: !32, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !12 = distinct !DISubprogram(name: "main", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !9, scopeLine: 11, file: !32, scope: !6, type: !13, retainedNodes: !19) !13 = !DISubroutineType(types: !14) !14 = !{!15, !15, !18} !15 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !18 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !19 = !{!21, !22} !21 = !DILocalVariable(name: "argc", line: 11, arg: 1, scope: !12, file: !6, type: !15) !22 = !DILocalVariable(name: "argv", line: 11, arg: 2, scope: !12, file: !6, type: !18) !23 = !DILocation(line: 6, column: 3, scope: !24) !24 = distinct !DILexicalBlock(line: 5, column: 16, file: !32, scope: !5) !25 = !DILocation(line: 7, column: 1, scope: !24) !26 = !DILocation(line: 11, column: 14, scope: !12) !27 = !DILocation(line: 11, column: 26, scope: !12) !28 = !DILocation(line: 12, column: 3, scope: !29) !29 = distinct !DILexicalBlock(line: 11, column: 34, file: !32, scope: !12) !30 = !DILocation(line: 13, column: 3, scope: !29) !31 = !DILocation(line: 14, column: 3, scope: !29) !32 = !DIFile(filename: "foo.c", directory: "/tmp") ; This test is simple to be cross platform (many targets don't yet have ; sufficiently good DWARF emission and/or dumping) ; CHECK: {{DW_TAG_compile_unit}} ; CHECK: {{foo\.c}} !33 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/typedef-arr-size.ll000066400000000000000000000030171506646577400254340ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Check that the size of vectors is translated source_filename = "test/DebugInfo/Generic/typedef-arr-size.ll" @x = dso_local global <16 x i32> zeroinitializer, align 16, !dbg !0 !llvm.dbg.cu = !{!8} !llvm.module.flags = !{!11, !12} !llvm.ident = !{!13} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "x", scope: null, file: !2, line: 2, type: !5, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "typedef-arr-size.cpp", directory: "/tmp/dbginfo") !3 = !{!4} !4 = !DISubrange(count: 16) ; CHECK: DICompositeType(tag: DW_TAG_array_type, {{.*}}, size: 512, flags: DIFlagVector, !5 = !DICompositeType(tag: DW_TAG_array_type, baseType: !6, size: 512, flags: DIFlagVector, elements: !3) !6 = !DIDerivedType(tag: DW_TAG_typedef, name: "data_t", file: !2, line: 42, baseType: !7) !7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !8 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.5.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !9, retainedTypes: !9, globals: !10, imports: !9) !9 = !{} !10 = !{!0} !11 = !{i32 2, !"Dwarf Version", i32 4} !12 = !{i32 1, !"Debug Info Version", i32 3} !13 = !{!"clang version 3.5.0 "} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/typedef.ll000066400000000000000000000030461506646577400237040ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll | llvm-dwarfdump -debug-info - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; From source: ; typedef void x; ; x *y; ; Check that a typedef with no DW_AT_type is produced. The absence of a type is used to imply the 'void' type. ; CHECK: DW_TAG_typedef ; CHECK-NOT: DW_AT_type ; CHECK: {{DW_TAG|NULL}} source_filename = "test/DebugInfo/Generic/typedef.ll" @y = global i8* null, align 8, !dbg !0 !llvm.dbg.cu = !{!5} !llvm.module.flags = !{!8, !9} !llvm.ident = !{!10} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "y", scope: null, file: !2, line: 2, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "typedef.cpp", directory: "/tmp/dbginfo") !3 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !4, size: 64, align: 64) !4 = !DIDerivedType(tag: DW_TAG_typedef, name: "x", file: !2, line: 1, baseType: null) !5 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.5.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !6, retainedTypes: !6, globals: !7, imports: !6) !6 = !{} !7 = !{!0} !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 1, !"Debug Info Version", i32 3} !10 = !{!"clang version 3.5.0 "} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/undef-func-call.ll000066400000000000000000000120341506646577400252040ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv ; This is a regression test for reported issue https://github.com/KhronosGroup/SPIRV-LLVM-Translator/issues/524. ; Test checks that reverse translation will not fail with assertion. ; Build from the following source with clang -c -emit-llvm -O0 -target spir64 -gline-tables-only ; float bar(int x); ; __kernel void foo(__global float* outPtr, int i) { ; #pragma clang loop unroll(enable) ; for (int j = 0; j < i; ++j) { ; outPtr[j] = bar(j); ; } ; } target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64" ; Function Attrs: convergent noinline norecurse nounwind optnone define dso_local spir_kernel void @foo(float addrspace(1)* %outPtr, i32 %i) #0 !dbg !9 !kernel_arg_addr_space !6 !kernel_arg_access_qual !11 !kernel_arg_type !12 !kernel_arg_base_type !12 !kernel_arg_type_qual !13 { entry: %outPtr.addr = alloca float addrspace(1)*, align 8 %i.addr = alloca i32, align 4 %j = alloca i32, align 4 store float addrspace(1)* %outPtr, float addrspace(1)** %outPtr.addr, align 8 store i32 %i, i32* %i.addr, align 4 store i32 0, i32* %j, align 4, !dbg !14 br label %for.cond, !dbg !15 for.cond: ; preds = %for.inc, %entry %0 = load i32, i32* %j, align 4, !dbg !16 %1 = load i32, i32* %i.addr, align 4, !dbg !17 %cmp = icmp slt i32 %0, %1, !dbg !18 br i1 %cmp, label %for.body, label %for.end, !dbg !19 for.body: ; preds = %for.cond %2 = load i32, i32* %j, align 4, !dbg !20 %call = call spir_func float @bar(i32 %2) #2, !dbg !21 %3 = load float addrspace(1)*, float addrspace(1)** %outPtr.addr, align 8, !dbg !22 %4 = load i32, i32* %j, align 4, !dbg !23 %idxprom = sext i32 %4 to i64, !dbg !22 %arrayidx = getelementptr inbounds float, float addrspace(1)* %3, i64 %idxprom, !dbg !22 store float %call, float addrspace(1)* %arrayidx, align 4, !dbg !24 br label %for.inc, !dbg !25 for.inc: ; preds = %for.body %5 = load i32, i32* %j, align 4, !dbg !26 %inc = add nsw i32 %5, 1, !dbg !26 store i32 %inc, i32* %j, align 4, !dbg !26 br label %for.cond, !dbg !19, !llvm.loop !27 for.end: ; preds = %for.cond ret void, !dbg !29 } ; Function Attrs: convergent declare dso_local spir_func float @bar(i32) #1 attributes #0 = { convergent noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { convergent "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #2 = { convergent } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !opencl.ocl.version = !{!6} !opencl.spir.version = !{!7} !llvm.ident = !{!8} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0 (https://github.com/llvm/llvm-project.git 6671a81bc71cc2635c5a10d6f688fea46ca4e5d6)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "loop.cl", directory: "/export/users/work/khr_spirv/llvm/build/bin") !2 = !{} !3 = !{i32 7, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{i32 1, i32 0} !7 = !{i32 1, i32 2} !8 = !{!"clang version 11.0.0"} !9 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !10 = !DISubroutineType(types: !2) !11 = !{!"none", !"none"} !12 = !{!"float*", !"int"} !13 = !{!"", !""} !14 = !DILocation(line: 5, column: 12, scope: !9) !15 = !DILocation(line: 5, column: 8, scope: !9) !16 = !DILocation(line: 5, column: 19, scope: !9) !17 = !DILocation(line: 5, column: 23, scope: !9) !18 = !DILocation(line: 5, column: 21, scope: !9) !19 = !DILocation(line: 5, column: 3, scope: !9) !20 = !DILocation(line: 6, column: 21, scope: !9) !21 = !DILocation(line: 6, column: 17, scope: !9) !22 = !DILocation(line: 6, column: 5, scope: !9) !23 = !DILocation(line: 6, column: 12, scope: !9) !24 = !DILocation(line: 6, column: 15, scope: !9) !25 = !DILocation(line: 7, column: 3, scope: !9) !26 = !DILocation(line: 5, column: 26, scope: !9) !27 = distinct !{!27, !19, !25, !28} !28 = !{!"llvm.loop.unroll.enable"} !29 = !DILocation(line: 8, column: 1, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/varargs.ll000066400000000000000000000102201506646577400237010ustar00rootroot00000000000000; REQUIRES: object-emission ; ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -filetype=obj %t.ll -o - | llvm-dwarfdump -v -debug-info - | FileCheck %s ; Test debug info for variadic function arguments. ; Created from tools/clang/tests/CodeGenCXX/debug-info-varargs.cpp with the ; function pointer removed. ; ; The ... parameter of variadic should be emitted as ; DW_TAG_unspecified_parameters. ; ; Normal variadic function. ; void b(int c, ...); ; ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "b" ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_unspecified_parameters ; ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name {{.*}} "a" ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_unspecified_parameters ; ; Variadic C++ member function. ; struct A { void a(int c, ...); } ; ; ModuleID = 'debug-info-varargs.cpp' source_filename = "debug-info-varargs.cpp" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64-unknown-unknown" %struct.A = type { i8 } ; Function Attrs: noinline nounwind optnone define spir_func void @_Z1biz(i32 %c, ...) #0 !dbg !6 { entry: %c.addr = alloca i32, align 4 %a = alloca %struct.A, align 1 store i32 %c, i32* %c.addr, align 4 call void @llvm.dbg.declare(metadata i32* %c.addr, metadata !11, metadata !DIExpression()), !dbg !12 call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !13, metadata !DIExpression()), !dbg !20 ret void, !dbg !21 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!5} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{!"clang version 8.0.0 "} !6 = distinct !DISubprogram(name: "b", linkageName: "_Z1biz", scope: !7, file: !7, line: 11, type: !8, isLocal: false, isDefinition: true, scopeLine: 11, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !7 = !DIFile(filename: "debug-info-varargs.cpp", directory: "/tmp") !8 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !9) !9 = !{null, !10, null} !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !DILocalVariable(name: "c", arg: 1, scope: !6, file: !7, line: 11, type: !10) !12 = !DILocation(line: 11, scope: !6) !13 = !DILocalVariable(name: "a", scope: !6, file: !7, line: 23, type: !14) !14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !7, line: 3, size: 8, flags: DIFlagTypePassByValue, elements: !15, identifier: "_ZTS1A") !15 = !{!16} !16 = !DISubprogram(name: "a", linkageName: "_ZN1A1aEiz", scope: !14, file: !7, line: 5, type: !17, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false) !17 = !DISubroutineType(types: !18) !18 = !{null, !19, !10, null} !19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !20 = !DILocation(line: 23, scope: !6) !21 = !DILocation(line: 29, scope: !6) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/Generic/version.ll000066400000000000000000000035461506646577400237360ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck < %t.ll %s --check-prefix=CHECK-LLVM ; RUN: llc -mtriple=%triple -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Make sure we are generating DWARF version 3 when module flag says so. ; CHECK: Compile Unit: length = {{.*}} version = 0x0003 ; CHECK-LLVM: !{i32 7, !"Dwarf Version", i32 3} define i32 @main() #0 !dbg !4 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval ret i32 0, !dbg !10 } attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!9, !11} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185475)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "CodeGen/dwarf-version.c", directory: "test") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "CodeGen/dwarf-version.c", directory: "test") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{i32 2, !"Dwarf Version", i32 3} !10 = !DILocation(line: 7, scope: !4) !11 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/InvalidSourceLanguageSPIRVtoLLVM.spvasm000066400000000000000000000033761506646577400276750ustar00rootroot00000000000000; This test checks that any invalid source language in the Compilation Unit instruction is mapped to ; DW_LANG_OpenCL and the original literal value is retained in the "Source Lang Literal" LLVM module flag metadata. ; REQUIRES: spirv-as ; RUN: spirv-as --target-env spv1.3 %s -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis | FileCheck %s ; SPIR-V ; Version: 1.1 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 16 ; Schema: 0 OpCapability Addresses OpCapability Kernel %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "OpenCL.DebugInfo.100" OpMemoryModel Physical64 OpenCL OpEntryPoint Kernel %5 "func" %7 = OpString "kernel_arg_type.func." %8 = OpString "/tmp/test.cl" %9 = OpString "//__CSK_MD5:18aa9ce738eaafc7b7b7181c19092815" %12 = OpString "func" %14 = OpString "" OpSource Unknown 0 OpName %entry "entry" OpModuleProcessed "Debug info producer: " %void = OpTypeVoid %4 = OpTypeFunction %void %10 = OpExtInst %void %2 DebugSource %8 %9 %11 = OpExtInst %void %2 DebugCompilationUnit 65536 5 %10 !0x0000002A %13 = OpExtInst %void %2 DebugInfoNone %15 = OpExtInst %void %2 DebugFunction %12 %13 %10 1 0 %11 %14 FlagIsDefinition|FlagPrototyped|FlagIsOptimized 2 %5 %13 %5 = OpFunction %void None %4 %entry = OpLabel OpReturn OpFunctionEnd ; CHECK: {{![0-9]+}} = !{i32 2, !"Source Lang Literal", [[LIST:![0-9]+]]} ; CHECK: [[LIST]] = !{[[ENTRY:![0-9]+]]} ; CHECK: [[ENTRY]] = !{[[CU:![0-9]+]], i32 42} ; CHECK: [[CU]] = distinct !DICompileUnit(language: DW_LANG_OpenCL SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/LocalAddressSpace.ll000066400000000000000000000073721506646577400242120ustar00rootroot00000000000000; Source: ;__kernel void foo(void) { ; __local int a; ;} ; clang -cc1 -triple spir -disable-llvm-passes -triple spir /work/tmp/tmp.cl -O0 -debug-info-kind=standalone -emit-llvm -o /work/llvm/projects/llvm-spirv/test/DebugInfo/LocalAddressSpace.ll ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv %t.bc -o - -spirv-text | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: cat %t.ll | FileCheck %s --check-prefix=CHECK-LLVM ; RUN: llc -mtriple=%triple -filetype=obj -O0 < %t.ll | llvm-dwarfdump -v -debug-info - | FileCheck %s ; CHECK-SPIRV: Variable {{[0-9]+}} [[foo_a:[0-9]+]] ; CHECK-SPIRV: DebugGlobalVariable {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} [[foo_a]] ; CHECK-LLVM: @foo.a = internal addrspace(3) global i32 undef, align 4, !dbg ![[a_dbg_expr:[0-9]+]] ; CHECK-LLVM: ![[a_dbg_expr]] = !DIGlobalVariableExpression(var: ![[a_dbg_var:[0-9]+]], ; CHECK-LLVM: ![[a_dbg_var]] = distinct !DIGlobalVariable(name: "a" ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_name {{.*}} = "a") ; CHECK-NEXT: DW_AT_type {{.*}} "int") ; CHECK-NEXT: DW_AT_decl_file {{.*}} ("/work{{[/\\]}}tmp{{[/\\]}}tmp.cl") ; CHECK-NEXT: DW_AT_decl_line {{.*}} (2) ; CHECK-NEXT: DW_AT_location [DW_FORM_exprloc] (DW_OP_addr 0x0) ; ModuleID = '/work/tmp/tmp.cl' source_filename = "/work/tmp/tmp.cl" target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir" @foo.a = internal addrspace(3) global i32 undef, align 4, !dbg !0 ; Function Attrs: convergent noinline nounwind optnone define spir_kernel void @foo() #0 !dbg !2 !kernel_arg_addr_space !8 !kernel_arg_access_qual !8 !kernel_arg_type !8 !kernel_arg_base_type !8 !kernel_arg_type_qual !8 { entry: ret void, !dbg !16 } attributes #0 = { convergent noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!6} !llvm.module.flags = !{!11, !12} !opencl.ocl.version = !{!13} !opencl.spir.version = !{!14} !llvm.ident = !{!15} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !10, isLocal: true, isDefinition: true) !2 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 1, type: !4, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !8) !3 = !DIFile(filename: "tmp/tmp.cl", directory: "/work") !4 = !DISubroutineType(cc: DW_CC_LLVM_OpenCLKernel, types: !5) !5 = !{null} !6 = distinct !DICompileUnit(language: DW_LANG_C99, file: !7, producer: "clang version 9.0.0 (https://llvm.org/git/clang 92470c6aadff9e614bfac44f48e6e1d430e5a32d) (https://llvm.org/git/llvm 461a7ee6493f997d6dc03ca0e80b6a7bd7943a83)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, globals: !9, nameTableKind: None) !7 = !DIFile(filename: "/work/tmp/", directory: "/work/llvm/build") !8 = !{} !9 = !{!0} !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{i32 1, !"wchar_size", i32 4} !13 = !{i32 1, i32 0} !14 = !{i32 1, i32 2} !15 = !{!"clang version 9.0.0 (https://llvm.org/git/clang 92470c6aadff9e614bfac44f48e6e1d430e5a32d) (https://llvm.org/git/llvm 461a7ee6493f997d6dc03ca0e80b6a7bd7943a83)"} !16 = !DILocation(line: 3, scope: !2) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/000077500000000000000000000000001506646577400225525ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/DIImportedEntity.ll000066400000000000000000000062431506646577400263050ustar00rootroot00000000000000; ModuleID = '/Volumes/Data/apple-internal/llvm/tools/clang/test/Modules/debug-info-moduleimport.m' ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-100 %t.bc -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefix CHECK-SPIRV ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-100 %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck %s --input-file %t.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 %t.bc -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefix CHECK-SPIRV ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: FileCheck %s --input-file %t.ll --check-prefix CHECK-LLVM target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; CHECK-SPIRV-DAG: ExtInstImport [[#EISId:]] "NonSemantic.Shader.DebugInfo ; CHECK-SPIRV-DAG: String [[#Name:]] "" ; CHECK-SPIRV-DAG: TypeInt [[#Int:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#Int]] [[#One:]] 1 ; CHECK-SPIRV-DAG: Constant [[#Int]] [[#Zero:]] 0 ; CHECK-SPIRV-DAG: Constant [[#Int]] [[#Five:]] 5 ; CHECK-SPIRV: ExtInst [[#]] [[#Source:]] [[#]] DebugSource ; CHECK-SPIRV: ExtInst [[#]] [[#CU:]] [[#]] DebugCompilationUnit ; CHECK-SPIRV: ExtInst [[#]] [[#Typedef:]] [[#]] DebugTypedef ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#EISId]] DebugImportedEntity [[#Name]] [[#One]] [[#Source]] [[#Typedef]] [[#Five]] [[#Zero]] [[#CU]] {{$}} ; CHECK-LLVM: ![[#CU:]] = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: ![[#File:]],{{.*}}isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: ![[#Import:]]) ; CHECK-LLVM: ![[#File]] = !DIFile(filename: "", directory: "/llvm/tools/clang/test/Modules") ; CHECK-LLVM: ![[#Import]] = !{![[#Entity:]]} ; CHECK-LLVM: ![[#Entity]] = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: ![[#CU]], entity: ![[#Typedef:]], file: ![[#File]], line: 5) ; CHECK-LLVM: ![[#Typedef]] = !DIDerivedType(tag: DW_TAG_typedef, name: "max_align_t", file: ![[#File]], baseType: ![[#]]) ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_structure_type, file: ![[#File]], line: 5, size: 256, flags: DIFlagFwdDecl, elements: ![[#]], identifier: "_ZTS11max_align_t") !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!6, !7} !llvm.ident = !{!8} !0 = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: !1, producer: "LLVM version 3.7.0", isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !3, sysroot: "/") !1 = !DIFile(filename: "/llvm/tools/clang/test/Modules/", directory: "/") !2 = !{} !3 = !{!4} !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !9, file: !1, line: 5) !6 = !{i32 2, !"Dwarf Version", i32 4} !7 = !{i32 2, !"Debug Info Version", i32 3} !8 = !{!"LLVM version 3.7.0"} !9 = !DIDerivedType(tag: DW_TAG_typedef, name: "max_align_t", file: !1, baseType: !10) !10 = !DICompositeType(tag: DW_TAG_structure_type, file: !1, line: 5, size: 256, flags: DIFlagFwdDecl, elements: !2, identifier: "_ZTS11max_align_t") SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/DebugFunction.cl000066400000000000000000000042571506646577400256360ustar00rootroot00000000000000// Check for 2 things: // - After round trip translation function definition has !dbg metadata attached // specifically if -gline-tables-only was used for Clang // - Parent operand of DebugFunction is DebugCompilationUnit, not an OpString, // even if in LLVM IR it points to a DIFile instead of DICompileUnit. // RUN: %clang_cc1 %s -cl-std=clc++ -emit-llvm-bc -triple spir -debug-info-kind=line-tables-only -O0 -o %t.bc // RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spv // RUN: llvm-spirv %t.spv -to-text -o %t.spt // RUN: FileCheck %s --input-file %t.spt --check-prefix=CHECK-SPIRV // RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv // RUN: llvm-spirv %t.spv -to-text -o %t.spt // RUN: FileCheck %s --input-file %t.spt --check-prefix=CHECK-SPIRV // RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM float foo(int i) { return i * 3.14; } void kernel k() { float a = foo(2); } // CHECK-SPIRV-DAG: String [[foo:[0-9]+]] "foo" // CHECK-SPIRV-DAG: String [[#EmptyStr:]] "" // CHECK-SPIRV-DAG: String [[k:[0-9]+]] "k" // CHECK-SPIRV-DAG: String [[#CV:]] {{.*}}clang version [[#]].[[#]].[[#]] // CHECK-SPIRV: [[#CU:]] [[#]] DebugCompilationUnit // CHECK-SPIRV: [[#FuncFoo:]] [[#]] DebugFunction [[foo]] {{.*}} [[#CU]] // CHECK-SPIRV: [[#FuncK:]] [[#]] DebugFunction [[k]] {{.*}} [[#CU]] // CHECK-SPIRV: DebugEntryPoint [[#FuncK]] [[#CU]] [[#CV]] [[#EmptyStr]] {{$}} // CHECK-SPIRV-NOT: DebugEntryPoint // CHECK-SPIRV-NOT: DebugFunctionDefinition // CHECK-SPIRV: Function {{[0-9]+}} [[#foo_id:]] // CHECK-SPIRV: DebugFunctionDefinition [[#FuncFoo]] [[#foo_id]] // CHECK-LLVM: define spir_func float @_Z3fooi(i32 %i) #{{[0-9]+}} !dbg ![[#foo_id:]] { // CHECK-SPIRV: Function {{[0-9]+}} [[#k_id:]] // CHECK-SPIRV: DebugFunctionDefinition [[#FuncK]] [[#k_id]] // CHECK-LLVM: define spir_kernel void @k() #{{[0-9]+}} !dbg ![[#k_id:]] // CHECK-LLVM: ![[#foo_id]] = distinct !DISubprogram(name: "foo" // CHECK-LLVM-SAME: spFlags: DISPFlagDefinition, // CHECK-LLVM: ![[#k_id]] = distinct !DISubprogram(name: "k" // CHECK-LLVM-SAME: spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/DebugInfoProducer.ll000066400000000000000000000106741506646577400264610ustar00rootroot00000000000000; Test checks debug info of producer is preserved from LLVM IR to spirv ; and spirv to LLVM IR translation. ; Original .cpp source: ; ; int main() { ; return 0; ; } ; Command line: ; ./clang -cc1 -debug-info-kind=standalone -v s.cpp -S -emit-llvm -triple spir ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV-200 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM-200 ; Check that we don't produce "producer" info for NonSemantic.Shader.DebugInfo.100 as it's not specification conformant ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefix CHECK-SPIRV-100 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix CHECK-LLVM-100 ; ModuleID = 's.bc' source_filename = "s.cpp" target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir" ; Function Attrs: noinline norecurse nounwind optnone define i32 @main() #0 !dbg !8 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval, align 4 ret i32 0, !dbg !13 } attributes #0 = { noinline norecurse nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5, !6} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!7} ; CHECK-LLVM-200: !DICompileUnit ; CHECK-LLVM-200-SAME: producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-LLVM-200-SAME: flags: "-O2" ; CHECK-LLVM-200-NOT: producer: "spirv" ; CHECK-LLVM-100: !DICompileUnit ; CHECK-LLVM-100-SAME: producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-LLVM-100-SAME: flags: "-O2" ; CHECK-SPIRV-200: String [[#ProducerId:]] "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-SPIRV-200: DebugBuildIdentifier [[#]] [[#]] ; CHECK-SPIRV-200: DebugStoragePath [[#]] ; CHECK-SPIRV-200: DebugCompilationUnit [[#]] [[#]] [[#]] [[#]] [[#ProducerId]] ; CHECK-SPIRV-200: DebugEntryPoint [[#]] [[#]] [[#ProducerId]] [[#]] {{$}} ; CHECK-SPIRV-100: String [[#ProducerId:]] "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)" ; CHECK-SPIRV-100: DebugBuildIdentifier [[#]] [[#]] ; CHECK-SPIRV-100: DebugStoragePath [[#]] ; CHECK-SPIRV-100-NOT: DebugCompilationUnit [[#]] [[#]] [[#]] [[#]] [[#ProducerId]] {{$}} ; CHECK-SPIRV-100: DebugEntryPoint [[#]] [[#]] [[#ProducerId]] [[#]] {{$}} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None, flags: "-O2") !1 = !DIFile(filename: "", directory: "oneAPI") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 1, !"ThinLTO", i32 0} !6 = !{i32 1, !"EnableSplitLTOUnit", i32 1} !7 = !{!"clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)"} !8 = distinct !DISubprogram(name: "main", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !0, retainedNodes: !2) !9 = !DIFile(filename: "s.cpp", directory: "C:\\") !10 = !DISubroutineType(types: !11) !11 = !{!12} !12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !13 = !DILocation(line: 2, column: 2, scope: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/DebugSourceContinued.ll000066400000000000000000020210271506646577400271670ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-100 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-100 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-200 ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-LLVM: !DIFile(filename: "t.c", directory: "/test", source: "A ; CHECK-LLVM-SAME-COUNT-200000: A ; CHECK-LLVM-SAME: MayThe4thBeWithYou ; CHECK-LLVM-SAME-COUNT-330000: A ; CHECK-SPIRV: String [[#Str:]] "A ; CHECK-SPIRV-SAME-COUNT-19999: A ; CHECK-SPIRV-SAME: MayThe4thBeWithYou ; CHECK-SPIRV-SAME-COUNT-62113: A ; CHECK-SPIRV: String [[#Str2:]] "A ; CHECK-SPIRV-SAME-COUNT-262130: A ; CHECK-SPIRV: String [[#Str3:]] "A ; CHECK-SPIRV-SAME-COUNT-5755: A ; CHECK-SPIRV-100: DebugSource [[#]] [[#Str]] ; CHECK-SPIRV-200: [[#NONE:]] [[#]] DebugInfoNone ; CHECK-SPIRV-200: DebugSource [[#]] [[#NONE]] [[#NONE]] [[#Str]] ; CHECK-SPIRV: DebugSourceContinued [[#Str2]] ; CHECK-SPIRV: DebugSourceContinued [[#Str3]] ; ModuleID = 't.c' source_filename = "t.c" ;target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" @t1 = global i32 1, align 4, !dbg !0 @t2 = global i32 0, align 4, !dbg !6 !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!11, !12, !13} !llvm.ident = !{!14} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "t1", scope: !2, file: !10, line: 1, type: !9, isLocal: false, isDefinition: true) !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (trunk 322159)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5) !3 = !DIFile(filename: "t.c", directory: "/test", source: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMayThe4thBeWithYouAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") !4 = !{} !5 = !{!0, !6} !6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression()) !7 = distinct !DIGlobalVariable(name: "t2", scope: !2, file: !8, line: 1, type: !9, isLocal: false, isDefinition: true) !8 = !DIFile(filename: "t2.h", directory: "/test", source: "22222222222222222222222222222222") !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !10 = !DIFile(filename: "t1.h", directory: "/test", source: "11111111111111111111111111111111") !11 = !{i32 2, !"Dwarf Version", i32 4} !12 = !{i32 2, !"Debug Info Version", i32 3} !13 = !{i32 1, !"wchar_size", i32 4} !14 = !{!"clang version 7.0.0 (trunk 322159)"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/000077500000000000000000000000001506646577400242025ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DIModule.ll000066400000000000000000000056471506646577400262110ustar00rootroot00000000000000; ModuleID = '/Volumes/Data/apple-internal/llvm/tools/clang/test/Modules/debug-info-moduleimport.m' ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 %t.bc -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-macosx %t.ll -accel-tables=Dwarf -o %t -filetype=obj ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s ; RUN: llvm-dwarfdump -verify %t ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 %t.bc -spirv-text -o %t.spt ; RUN: FileCheck %s --input-file %t.spt --check-prefix CHECK-SPIRV ; CHECK: DW_TAG_compile_unit ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_module ; CHECK-NEXT: DW_AT_name {{.*}}"DebugModule" ; CHECK-NEXT: DW_AT_LLVM_config_macros {{.*}}"-DMODULES=0" ; CHECK-NEXT: DW_AT_LLVM_include_path {{.*}}"/llvm/tools/clang/test/Modules/Inputs" ; CHECK-NEXT: DW_AT_LLVM_apinotes {{.*}}"m.apinotes" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; CHECK-SPIRV-DAG: ExtInstImport [[#EISId:]] "NonSemantic.Shader.DebugInfo.200" ; CHECK-SPIRV: String [[#FileName:]] "/llvm/tools/clang/test/Modules/" ; CHECK-SPIRV: String [[#EmptyStr:]] "" ; CHECK-SPIRV: String [[#Name:]] "DebugModule" ; CHECK-SPIRV: String [[#Defines:]] "-DMODULES=0" ; CHECK-SPIRV: String [[#IncludePath:]] "/llvm/tools/clang/test/Modules/Inputs" ; CHECK-SPIRV: String [[#ApiNotes:]] "m.apinotes" ; CHECK-SPIRV: TypeInt [[#TypeInt32:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#TypeInt32]] [[#Constant0:]] 0 ; CHECK-SPIRV-DAG: Constant [[#TypeInt32]] [[#Version:]] 65536 ; CHECK-SPIRV-DAG: Constant [[#TypeInt32]] [[#DWARF:]] 4 ; CHECK-SPIRV: ExtInst [[#]] [[#Source:]] [[#]] DebugSource [[#FileName]] ; CHECK-SPIRV: ExtInst [[#]] [[#Parent:]] [[#]] DebugCompilationUnit [[#Version]] [[#DWARF]] ; CHECK-SPIRV: ExtInst [[#]] [[#SourceEmpty:]] [[#]] DebugSource [[#EmptyStr]] ; CHECK-SPIRV: ExtInst [[#]] [[#Module:]] [[#]] DebugModule [[#Name]] [[#SourceEmpty]] [[#Constant0]] [[#Parent]] [[#Defines]] [[#IncludePath]] [[#ApiNotes]] [[#Constant0]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugImportedEntity [[#]] [[#]] [[#Source]] [[#Module]] !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!6, !7} !llvm.ident = !{!8} !0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "LLVM version 3.7.0", isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !3, sysroot: "/") !1 = !DIFile(filename: "/llvm/tools/clang/test/Modules/", directory: "/") !2 = !{} !3 = !{!4} !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !5, file: !1, line: 5) !5 = !DIModule(scope: null, name: "DebugModule", configMacros: "-DMODULES=0", includePath: "/llvm/tools/clang/test/Modules/Inputs", apinotes: "m.apinotes") !6 = !{i32 2, !"Dwarf Version", i32 4} !7 = !{i32 2, !"Debug Info Version", i32 3} !8 = !{!"LLVM version 3.7.0"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DebugInfoStringType.ll000066400000000000000000000267371506646577400304450ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o %t.spt --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -to-binary %t.spt -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; CHECK-SPIRV: ExtInstImport [[#EISId:]] "NonSemantic.Shader.DebugInfo.200" ; CHECK-SPIRV: String [[#StrGreet:]] ".str.GREETING" ; CHECK-SPIRV: String [[#StrChar1:]] "CHARACTER_1" ; CHECK-SPIRV: String [[#StrChar2:]] "CHARACTER_2" ; CHECK-SPIRV: String [[#StrChar3:]] "CHARACTER_3" ; CHECK-SPIRV: TypeInt [[#TypeInt:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#ConstZero:]] 0{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const1:]] 1{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const2:]] 2{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const6:]] 6{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const7:]] 7{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const9:]] 9{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const12:]] 12{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const23:]] 23{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const24:]] 24{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const27:]] 27{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const28:]] 28{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const36:]] 36{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const37:]] 37{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const80:]] 80{{[[:space:]]}} ; CHECK-SPIRV: TypeVoid [[#TypeVoid:]] ; CHECK-SPIRV: [[#DINoneId:]] [[#EISId]] DebugInfoNone ; CHECK-SPIRV: [[#DataLocExpr:]] [[#EISId]] DebugExpression [[#]] [[#]] {{$}} ; CHECK-SPIRV: [[#LengthAddrExpr:]] [[#EISId]] DebugExpression [[#]] [[#]] {{$}} ; DebugTypeString NameId BaseTyId DataLocId SizeId LengthAddrId ; CHECK-SPIRV: [[#EISId]] DebugTypeString [[#StrGreet]] [[#DINoneId]] [[#DataLocExpr]] [[#ConstZero]] [[#LengthAddrExpr]] ; CHECK-SPIRV: [[#EISId]] DebugTypeString [[#StrChar1]] [[#DINoneId]] [[#DINoneId]] [[#Const80]] [[#DINoneId]] ; CHECK-SPIRV-COUNT-2: [[#LengthAddrVar:]] [[#EISId]] DebugLocalVariable ; CHECK-SPIRV-NEXT: [[#EISId]] DebugTypeString [[#StrChar2]] [[#DINoneId]] [[#DINoneId]] [[#ConstZero]] [[#LengthAddrVar]] ; CHECK-SPIRV-COUNT-3: [[#LengthAddrVar1:]] [[#EISId]] DebugLocalVariable ; CHECK-SPIRV-NEXT: [[#EISId]] DebugTypeString [[#StrChar3]] [[#DINoneId]] [[#DINoneId]] [[#ConstZero]] [[#LengthAddrVar1]] ; CHECK-SPIRV-COUNT-4: [[#LengthAddrVar2:]] [[#EISId]] DebugLocalVariable ; CHECK-SPIRV-NEXT: [[#EISId]] DebugTypeString [[#StrChar2]] [[#DINoneId]] [[#DINoneId]] [[#ConstZero]] [[#LengthAddrVar2]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const6]] [[#Const6]] [[#Const23]] [[#Const24]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#ConstZero]] [[#ConstZero]] [[#ConstZero]] [[#Const1]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const9]] [[#Const9]] [[#Const27]] [[#Const28]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const9]] [[#Const9]] [[#Const36]] [[#Const37]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const7]] [[#Const7]] [[#Const1]] [[#Const2]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#ConstZero]] [[#ConstZero]] [[#ConstZero]] [[#Const1]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const9]] [[#Const9]] [[#Const27]] [[#Const28]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const9]] [[#Const9]] [[#Const36]] [[#Const37]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Const12]] [[#Const12]] [[#Const1]] [[#Const2]] ; CHECK-LLVM-DAG: !DICompileUnit(language: DW_LANG_Fortran95 ; CHECK-LLVM-DAG: ![[#Scope_hello_world:]] = distinct !DISubprogram(name: "hello_world", linkageName: "MAIN__" ; CHECK-LLVM-DAG: !DILocation(line: 6, column: 23, scope: ![[#Scope_hello_world]] ; CHECK-LLVM-DAG: !DIStringType(name: "CHARACTER_1", size: 80) ; CHECK-LLVM-DAG: !DIStringType(name: ".str.GREETING", stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8), stringLocationExpression: !DIExpression(DW_OP_push_object_address, DW_OP_deref)) ; CHECK-LLVM-DAG: ![[#Scope_print_greeting:]] = distinct !DISubprogram(name: "print_greeting", linkageName: "print_greeting_" ; CHECK-LLVM-DAG: ![[#StrLenMD:]] = !DILocalVariable(name: "STRING1.len", scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-DAG: !DIStringType(name: "CHARACTER_2", stringLength: ![[#StrLenMD]]) ; CHECK-LLVM-DAG: ![[#StrLenMD1:]] = !DILocalVariable(name: "STRING2.len", scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-DAG: !DIStringType(name: "CHARACTER_3", stringLength: ![[#StrLenMD1]]) ; CHECK-LLVM: !DILocation(line: 9, column: 27, scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-NEXT: !DILocation(line: 9, column: 36, scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-NEXT: !DILocation(line: 7, column: 1, scope: ![[#Scope_hello_world]] ; CHECK-LLVM-NEXT: !DILocation(line: 0, scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-NEXT: !DILocation(line: 9, column: 27, scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-NEXT: !DILocation(line: 9, column: 36, scope: ![[#Scope_print_greeting]] ; CHECK-LLVM-NEXT: !DILocation(line: 12, column: 1, scope: ![[#Scope_print_greeting]] target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" %"QNCA_a0$i8*$rank0$" = type { ptr, i64, i64, i64, i64, i64 } @strlit = internal unnamed_addr constant [5 x i8] c"HELLO" @strlit.1 = internal unnamed_addr constant [3 x i8] c"TOM" @"hello_world_$GREETING" = internal global %"QNCA_a0$i8*$rank0$" zeroinitializer, !dbg !2 @"hello_world_$NAME" = internal global [10 x i8] zeroinitializer, align 1, !dbg !10 @0 = internal unnamed_addr constant i32 65536, align 4 @1 = internal unnamed_addr constant i32 2, align 4 @strlit.2 = internal unnamed_addr constant [2 x i8] c", " ; Function Attrs: nounwind uwtable define void @MAIN__() local_unnamed_addr #0 !dbg !4{ %"hello_world_$GREETING_fetch.16" = load ptr, ptr @"hello_world_$GREETING", align 16, !dbg !20 %fetch.15 = load i64, ptr getelementptr inbounds (%"QNCA_a0$i8*$rank0$", ptr @"hello_world_$GREETING", i64 0, i32 1), align 8, !dbg !20 call void @llvm.dbg.value(metadata i64 %fetch.15, metadata !24, metadata !DIExpression()), !dbg !21 call void @llvm.dbg.value(metadata i64 %fetch.15, metadata !31, metadata !DIExpression()), !dbg !21 call void @llvm.dbg.value(metadata i64 10, metadata !28, metadata !DIExpression()), !dbg !21 call void @llvm.dbg.value(metadata i64 10, metadata !32, metadata !DIExpression()), !dbg !21 call void @llvm.dbg.declare(metadata ptr %"hello_world_$GREETING_fetch.16", metadata !26, metadata !DIExpression()), !dbg !36 call void @llvm.dbg.declare(metadata ptr @"hello_world_$NAME", metadata !29, metadata !DIExpression()), !dbg !37 ret void, !dbg !38 } ; Function Attrs: nofree nounwind uwtable define void @print_greeting_(ptr noalias readonly %"print_greeting_$STRING1", ptr noalias readonly %"print_greeting_$STRING2", i64 %"STRING1.len$val", i64 %"STRING2.len$val") local_unnamed_addr #1 !dbg !22 { alloca_1: call void @llvm.dbg.value(metadata i64 %"STRING1.len$val", metadata !24, metadata !DIExpression()), !dbg !39 call void @llvm.dbg.value(metadata i64 %"STRING1.len$val", metadata !31, metadata !DIExpression()), !dbg !39 call void @llvm.dbg.value(metadata i64 %"STRING2.len$val", metadata !28, metadata !DIExpression()), !dbg !39 call void @llvm.dbg.value(metadata i64 %"STRING2.len$val", metadata !32, metadata !DIExpression()), !dbg !39 call void @llvm.dbg.declare(metadata ptr %"print_greeting_$STRING1", metadata !26, metadata !DIExpression()), !dbg !40 call void @llvm.dbg.declare(metadata ptr %"print_greeting_$STRING2", metadata !29, metadata !DIExpression()), !dbg !41 ret void, !dbg !42 } ; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) #3 attributes #0 = { nounwind uwtable } attributes #1 = { nofree nounwind uwtable} attributes #2 = { mustprogress nocallback nofree nosync nounwind speculatable willreturn } attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn } !llvm.module.flags = !{!18, !19} !llvm.dbg.cu = !{!8} !2 = !DIGlobalVariableExpression(var: !3, expr: !DIExpression()) !3 = distinct !DIGlobalVariable(name: "greeting", linkageName: "hello_world_$GREETING", scope: !4, file: !5, line: 3, type: !14, isLocal: true, isDefinition: true) !4 = distinct !DISubprogram(name: "hello_world", linkageName: "MAIN__", scope: !5, file: !5, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !8, retainedNodes: !13) !5 = !DIFile(filename: "hello.f90", directory: "/dev/null") !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !5, producer: "fortran", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !9, splitDebugInlining: false, nameTableKind: None) !9 = !{!2, !10} !10 = !DIGlobalVariableExpression(var: !11, expr: !DIExpression()) !11 = distinct !DIGlobalVariable(name: "name", linkageName: "hello_world_$NAME", scope: !4, file: !5, line: 2, type: !12, isLocal: true, isDefinition: true) !12 = !DIStringType(name: "CHARACTER_1", size: 80) !13 = !{} !14 = !DIStringType(name: ".str.GREETING", stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8), stringLocationExpression: !DIExpression(DW_OP_push_object_address, DW_OP_deref)) !18 = !{i32 2, !"Debug Info Version", i32 3} !19 = !{i32 2, !"Dwarf Version", i32 4} !20 = !DILocation(line: 6, column: 23, scope: !4) !21 = !DILocation(line: 0, scope: !22, inlinedAt: !33) !22 = distinct !DISubprogram(name: "print_greeting", linkageName: "print_greeting_", scope: !5, file: !5, line: 9, type: !6, scopeLine: 9, spFlags: DISPFlagDefinition, unit: !8, retainedNodes: !23) !23 = !{!24, !26, !28, !29, !31, !32} !24 = !DILocalVariable(name: "STRING1.len", scope: !22, type: !25, flags: DIFlagArtificial) !25 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed) !26 = !DILocalVariable(name: "string1", arg: 1, scope: !22, file: !5, line: 9, type: !27) !27 = !DIStringType(name: "CHARACTER_2", stringLength: !24) !28 = !DILocalVariable(name: "STRING2.len", scope: !22, type: !25, flags: DIFlagArtificial) !29 = !DILocalVariable(name: "string2", arg: 2, scope: !22, file: !5, line: 9, type: !30) !30 = !DIStringType(name: "CHARACTER_3", stringLength: !28) !31 = !DILocalVariable(name: "_string1", arg: 3, scope: !22, type: !25, flags: DIFlagArtificial) !32 = !DILocalVariable(name: "_string2", arg: 4, scope: !22, type: !25, flags: DIFlagArtificial) !33 = distinct !DILocation(line: 0, scope: !34, inlinedAt: !35) !34 = distinct !DISubprogram(name: "print_greeting_.t60p.t61p.t3v.t3v", linkageName: "print_greeting_.t60p.t61p.t3v.t3v", scope: !5, file: !5, type: !6, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !8, retainedNodes: !13, targetFuncName: "print_greeting_") !35 = distinct !DILocation(line: 6, column: 8, scope: !4) !36 = !DILocation(line: 9, column: 27, scope: !22, inlinedAt: !33) !37 = !DILocation(line: 9, column: 36, scope: !22, inlinedAt: !33) !38 = !DILocation(line: 7, column: 1, scope: !4) !39 = !DILocation(line: 0, scope: !22) !40 = !DILocation(line: 9, column: 27, scope: !22) !41 = !DILocation(line: 9, column: 36, scope: !22) !42 = !DILocation(line: 12, column: 1, scope: !22) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DebugInfoSubrange.ll000066400000000000000000000224121506646577400300650ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o %t.spt --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -to-binary %t.spt -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; CHECK-SPIRV: ExtInstImport [[#EISId:]] "NonSemantic.Shader.DebugInfo.200" ; CHECK-SPIRV: String [[#LocalVarNameId:]] "A$1$upperbound" ; CHECK-SPIRV-DAG: TypeInt [[#TyInt32Id:]] 32 0 ; CHECK-SPIRV-DAG: TypeInt [[#TyInt64Id:]] 64 0 ; CHECK-NOT: THIS LINE IS USED TO SEPARATE DAGs ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant15Id:]] 15{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant24Id:]] 24{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant25Id:]] 25{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant27Id:]] 27{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant33Id:]] 33{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant34Id:]] 34{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant67Id:]] 67{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant68Id:]] 68{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt64Id]] [[#Constant1Id:]] 1 0 ; CHECK-SPIRV-DAG: Constant [[#TyInt64Id]] [[#Constant1000Id:]] 1000 0 ; CHECK-SPIRV: [[#DINoneId:]] [[#EISId]] DebugInfoNone ; CHECK-SPIRV: [[#DebugFuncId:]] [[#EISId]] DebugFunction ; CHECK-SPIRV: [[#LocalVarId:]] [[#EISId]] DebugLocalVariable [[#LocalVarNameId]] [[#]] [[#]] [[#]] [[#]] [[#DebugFuncId]] ; CHECK-SPIRV: [[#DebugTypeTemplate:]] [[#EISId]] DebugTypeTemplate [[#DebugFuncId]] ; CHECK-SPIRV: [[#EISId]] DebugTypeSubrange [[#Constant1Id]] [[#LocalVarId]] [[#DINoneId]] {{$}} ; CHECK-SPIRV: [[#DIExprId:]] [[#EISId]] DebugExpression ; CHECK-SPIRV: [[#EISId]] DebugTypeSubrange [[#DIExprId]] [[#DIExprId]] [[#DINoneId]] {{$}} ; CHECK-SPIRV: [[#EISId]] DebugTypeSubrange [[#Constant1Id]] [[#DINoneId]] [[#Constant1000Id]] {{$}} ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant15Id]] [[#Constant15Id]] [[#Constant67Id]] [[#Constant68Id]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant27Id]] [[#Constant27Id]] [[#Constant24Id]] [[#Constant25Id]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant34Id]] [[#Constant34Id]] [[#Constant33Id]] [[#Constant34Id]] ; CHECK-LLVM: ![[#Scope_A:]] = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split" ; CHECK-LLVM: [[#Subrange1:]] = !DISubrange(lowerBound: 1, upperBound: ![[#UpperBound:]]) ; CHECK-LLVM: [[#UpperBound]] = !DILocalVariable(name: "A$1$upperbound" ; CHECK-LLVM: !DILocation(line: 15, column: 67, scope: ![[#Scope_A]] ; CHECK-LLVM: ![[#Scope_B:]] = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split" ; CHECK-LLVM: !DISubrange(lowerBound: !DIExpression(), upperBound: !DIExpression()) ; CHECK-LLVM: !DILocation(line: 15, column: 67, scope: ![[#Scope_B]] ; CHECK-LLVM: ![[#Scope_C:]] = distinct !DISubprogram(name: "test_target_map_array_default_IP_test_array_map_no_map_type_.DIR.OMP.TARGET.340.split" ; CHECK-LLVM: !DISubrange(count: 1000, lowerBound: 1) ; CHECK-LLVM: !DILocation(line: 27, column: 24, scope: ![[#Scope_C]] ; CHECK-LLVM: ![[#Scope_D:]] = distinct !DISubprogram(name: "test" ; CHECK-LLVM: !DILocation(line: 34, column: 33, scope: ![[#Scope_D]] ; ModuleID = 'DebugInfoSubrangeUpperBound.bc' target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024" target triple = "spir64-unknown-unknown" %structtype = type { [72 x i1] } %"QNCA_a0$float" = type { float addrspace(4)*, i64, i64, i64, i64, i64, [1 x %structtype2] } %structtype2 = type { i64, i64, i64 } ; Function Attrs: noinline nounwind define spir_kernel void @__omp_offloading_811_198142f_random_fill_sp_l25(i32 addrspace(1)* noalias %0, %structtype* byval(%structtype) %"ascast$val", [1000 x i32] addrspace(1)* noalias %"ascastB$val") #0 !kernel_arg_addr_space !9 !kernel_arg_access_qual !10 !kernel_arg_type !11 !kernel_arg_type_qual !12 !kernel_arg_base_type !11 { newFuncRoot: %.ascast = bitcast %structtype* %"ascast$val" to %"QNCA_a0$float"* call void @llvm.dbg.value(metadata %"QNCA_a0$float"* %.ascast, metadata !13, metadata !DIExpression(DW_OP_deref)), !dbg !27 call void @llvm.dbg.value(metadata %"QNCA_a0$float"* %.ascast, metadata !28, metadata !DIExpression(DW_OP_deref)), !dbg !42 call void @llvm.dbg.value(metadata [1000 x i32] addrspace(1)* %"ascastB$val", metadata !47, metadata !DIExpression(DW_OP_deref)), !dbg !51 call void @llvm.dbg.value(metadata i32 addrspace(1)* %0, metadata !54, metadata !DIExpression(DW_OP_deref)), !dbg !59 ret void } ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { noinline nounwind } attributes #1 = { nofree nosync nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !spirv.MemoryModel = !{!4} !opencl.enable.FP_CONTRACT = !{} !spirv.Source = !{!5} !opencl.spir.version = !{!6} !opencl.ocl.version = !{!6} !opencl.used.extensions = !{!7} !opencl.used.optional.core.features = !{!7} !spirv.Generator = !{!8} !0 = !{i32 7, !"Dwarf Version", i32 4} !1 = !{i32 2, !"Debug Info Version", i32 3} !2 = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: !3, producer: "Fortran", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) !3 = !DIFile(filename: "f.f90", directory: "Fortran") !4 = !{i32 2, i32 2} !5 = !{i32 4, i32 200000} !6 = !{i32 2, i32 0} !7 = !{} !8 = !{i16 6, i16 14} !9 = !{i32 0} !10 = !{!"none"} !11 = !{!"structtype"} !12 = !{!""} !13 = !DILocalVariable(name: "a", scope: !14, file: !3, line: 15, type: !18) !14 = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split", scope: null, file: !3, line: 25, type: !15, scopeLine: 25, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !17) !15 = !DISubroutineType(types: !16) !16 = !{null} !17 = !{!13} !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64) !19 = !DICompositeType(tag: DW_TAG_array_type, baseType: !20, size: 32, elements: !21) !20 = !DIBasicType(name: "REAL*4", size: 32, encoding: DW_ATE_float) !21 = !{!22} !22 = !DISubrange(lowerBound: 1, upperBound: !23) !23 = !DILocalVariable(name: "A$1$upperbound", scope: !24, type: !26, flags: DIFlagArtificial) !24 = distinct !DISubprogram(name: "random_fill_sp", linkageName: "random_fill_sp", scope: null, file: !3, line: 15, type: !15, scopeLine: 15, spFlags: DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !25) !25 = !{!23} !26 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed) !27 = !DILocation(line: 15, column: 67, scope: !14) !28 = !DILocalVariable(name: "a", scope: !29, file: !3, line: 15, type: !33) !29 = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split", scope: null, file: !3, line: 25, type: !30, scopeLine: 25, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !32) !30 = !DISubroutineType(types: !31) !31 = !{null} !32 = !{!28} !33 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !34, size: 64) !34 = !DICompositeType(tag: DW_TAG_array_type, baseType: !35, size: 32, elements: !36) !35 = !DIBasicType(name: "REAL*4", size: 32, encoding: DW_ATE_float) !36 = !{!37} !37 = !DISubrange(lowerBound: !DIExpression(), upperBound: !DIExpression()) !38 = !DILocalVariable(name: "A$1$upperbound", scope: !39, type: !41, flags: DIFlagArtificial) !39 = distinct !DISubprogram(name: "random_fill_sp", linkageName: "random_fill_sp", scope: null, file: !3, line: 15, type: !30, scopeLine: 15, spFlags: DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !40) !40 = !{!38} !41 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed) !42 = !DILocation(line: 15, column: 67, scope: !29) !43 = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) !44 = !{} !45 = !DISubroutineType(types: !44) !46 = distinct !DISubprogram(name: "test_target_map_array_default_IP_test_array_map_no_map_type_.DIR.OMP.TARGET.340.split", scope: !3, file: !3, line: 32, type: !45, scopeLine: 32, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2) !47 = !DILocalVariable(name: "compute_array", scope: !46, file: !3, line: 27, type: !48) !48 = !DICompositeType(tag: DW_TAG_array_type, baseType: !43, elements: !49) !49 = !{!50} !50 = !DISubrange(count: 1000, lowerBound: 1) !51 = !DILocation(line: 27, column: 24, scope: !46) !52 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 51, type: !53, scopeLine: 51, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2) !53 = !DISubroutineType(types: !7) !54 = !DILocalVariable(name: "isHost", scope: !52, file: !3, line: 34, type: !55) !55 = !DICompositeType(tag: DW_TAG_array_type, baseType: !56, elements: !57) !56 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !57 = !{!58} !58 = !DISubrange(count: -1) !59 = !DILocation(line: 34, column: 33, scope: !52) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DebugInfoTargetFunction.ll000066400000000000000000000112321506646577400312510ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o %t.spt --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -to-binary %t.spt -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; CHECK-SPIRV-DAG: ExtInstImport [[#EISId:]] "NonSemantic.Shader.DebugInfo.200" ; CHECK-SPIRV-DAG: String [[#Func:]] "foo_wrapper" ; CHECK-SPIRV-DAG: String [[#TargetFunc:]] "_Z3foov" ; CHECK-SPIRV-DAG: TypeInt [[#TyInt32Id:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant1Id:]] 1{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant2Id:]] 2{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant4Id:]] 4{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant5Id:]] 5{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant6Id:]] 6{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant8Id:]] 8{{[[:space:]]}} ; CHECK-SPIRV-DAG: Constant [[#TyInt32Id]] [[#Constant9Id:]] 9{{[[:space:]]}} ; CHECK-SPIRV-DAG: ExtInst [[#]] [[#DebugNone:]] [[#]] DebugInfoNone ; CHECK-SPIRV-DAG: ExtInst [[#]] [[#]] [[#]] DebugFunction [[#Func]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#DebugNone]] [[#TargetFunc]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant4Id]] [[#Constant4Id]] [[#Constant5Id]] [[#Constant6Id]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant5Id]] [[#Constant5Id]] [[#Constant1Id]] [[#Constant2Id]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant8Id]] [[#Constant8Id]] [[#Constant5Id]] [[#Constant6Id]] ; CHECK-SPIRV: [[#EISId]] DebugLine [[#]] [[#Constant9Id]] [[#Constant9Id]] [[#Constant1Id]] [[#Constant2Id]] ; CHECK-LLVM: define spir_func void @_Z11foo_wrapperv() {{.*}} !dbg ![[#DbgSubProg:]] { ; CHECK-LLVM: ![[#Scope_foo_wrapper:]] = distinct !DISubprogram(name: "foo_wrapper", linkageName: "_Z11foo_wrapperv", scope: null, file: ![[#]], line: 3, type: ![[#]], scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: ![[#]], templateParams: ![[#]], retainedNodes: ![[#]], targetFuncName: "_Z3foov") ; CHECK-LLVM: !DILocation(line: 4, column: 5, scope: ![[#Scope_foo_wrapper]] ; CHECK-LLVM: !DILocation(line: 5, column: 1, scope: ![[#Scope_foo_wrapper]] ; CHECK-LLVM: ![[#Scope_boo:]] = distinct !DISubprogram(name: "boo", linkageName: "_Z3boov" ; CHECK-LLVM: !DILocation(line: 8, column: 5, scope: ![[#Scope_boo]] ; CHECK-LLVM: !DILocation(line: 9, column: 1, scope: ![[#Scope_boo]] ; ModuleID = 'example.bc' target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024" target triple = "spir64-unknown-unknown" define spir_func void @_Z11foo_wrapperv() !dbg !10 { call void @_Z3foov(), !dbg !15 ret void, !dbg !16 } declare spir_func void @_Z3foov() define spir_func void @_Z3boov() !dbg !17 { call void @_Z11foo_wrapperv(), !dbg !18 ret void, !dbg !19 } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} !llvm.ident = !{!9} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git 88bd2601c013e349fa907b3f878312a94e16e9f6)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "/app/example.cpp", directory: "/app") !2 = !{i32 7, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 8, !"PIC Level", i32 2} !6 = !{i32 7, !"PIE Level", i32 2} !7 = !{i32 7, !"uwtable", i32 2} !8 = !{i32 7, !"frame-pointer", i32 2} !9 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git 88bd2601c013e349fa907b3f878312a94e16e9f6)"} !10 = distinct !DISubprogram(name: "foo_wrapper", linkageName: "_Z11foo_wrapperv", scope: !11, file: !11, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14, targetFuncName: "_Z3foov") !11 = !DIFile(filename: "example.cpp", directory: "/app") !12 = !DISubroutineType(types: !13) !13 = !{null} !14 = !{} !15 = !DILocation(line: 4, column: 5, scope: !10) !16 = !DILocation(line: 5, column: 1, scope: !10) !17 = distinct !DISubprogram(name: "boo", linkageName: "_Z3boov", scope: !11, file: !11, line: 7, type: !12, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14) !18 = !DILocation(line: 8, column: 5, scope: !17) !19 = !DILocation(line: 9, column: 1, scope: !17) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DebugLinePriority.spt000066400000000000000000000050011506646577400303260ustar00rootroot00000000000000; RUN: llvm-spirv -to-binary %s -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; CHECK-LLVM: %[[#Var:]] = load i32, i32* @_ZN5Outer5Inner6globalE, align 4, !dbg ![[#LineLoc:]] ; CHECK-LLVM: %inc = add nsw i32 %[[#Var]], 1, !dbg ![[#DebugLineLoc:]] ; CHECK-LLVM: ![[#LineLoc]] = !DILocation(line: 357, column: 113, scope: ![[#Scope:]]) ; CHECK-LLVM: ![[#DebugLineLoc]] = !DILocation(line: 8, column: 16, scope: ![[#Scope]]) 119734787 66560 393230 54 0 2 Capability Addresses 2 Capability Linkage 2 Capability Kernel 8 Extension "SPV_KHR_non_semantic_info" 5 ExtInstImport 1 "OpenCL.std" 11 ExtInstImport 2 "NonSemantic.Shader.DebugInfo.200" 3 MemoryModel 2 2 10 String 14 "/path/to/inlined-namespace.cxx" 3 String 16 "0" 3 String 18 "" 4 String 23 "clang" 3 String 25 "int" 3 String 30 "foo" 4 String 31 "_Z3foov" 4 String 37 "Outer" 4 String 41 "Inner" 4 String 44 "global" 8 String 45 "_ZN5Outer5Inner6globalE" 3 Source 0 0 8 Name 6 "_ZN5Outer5Inner6globalE" 4 Name 9 "_Z3foov" 4 Name 10 "entry" 3 Name 13 "inc" 10 Decorate 6 LinkageAttributes "_ZN5Outer5Inner6globalE" Export 4 Decorate 6 Alignment 4 6 Decorate 9 LinkageAttributes "_Z3foov" Export 3 Decorate 13 NoSignedWrap 4 TypeInt 3 32 0 4 Constant 3 5 0 4 Constant 3 12 1 4 Constant 3 20 65536 4 Constant 3 21 4 4 Constant 3 22 217 4 Constant 3 26 32 4 Constant 3 32 7 4 Constant 3 33 136 4 Constant 3 46 3 4 Constant 3 47 8 4 Constant 3 50 16 4 Constant 3 52 9 4 TypePointer 4 7 3 2 TypeVoid 7 3 TypeFunction 8 7 2 TypeBool 38 5 Variable 4 6 7 5 3 ConstantTrue 38 39 3 ConstantFalse 38 42 6 ExtInst 7 15 2 DebugSource 14 7 ExtInst 7 17 2 DebugBuildIdentifier 16 12 6 ExtInst 7 19 2 DebugStoragePath 18 10 ExtInst 7 24 2 DebugCompilationUnit 20 21 15 22 23 8 ExtInst 7 27 2 DebugTypeBasic 25 26 21 5 ExtInst 7 28 2 DebugInfoNone 7 ExtInst 7 29 2 DebugTypeFunction 5 28 15 ExtInst 7 34 2 DebugFunction 30 29 15 32 5 24 31 33 32 28 6 ExtInst 7 36 2 DebugSource 18 11 ExtInst 7 40 2 DebugLexicalBlock 36 5 5 24 37 39 11 ExtInst 7 43 2 DebugLexicalBlock 36 5 5 40 41 42 14 ExtInst 7 48 2 DebugGlobalVariable 44 27 15 46 5 43 45 6 47 5 Function 7 9 2 8 2 Label 10 7 ExtInst 7 35 2 DebugFunctionDefinition 34 9 6 ExtInst 7 49 2 DebugScope 34 4 Line 14 357 113 6 Load 3 11 6 2 4 10 ExtInst 7 51 2 DebugLine 14 47 47 50 33 5 IAdd 3 13 11 12 5 ExtInst 7 511 2 DebugNoLine 5 ISub 3 137 11 12 1 NoLine 5 Store 6 13 2 4 1 Return 1 FunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/DebugTypeDef.ll000066400000000000000000000057741506646577400270570ustar00rootroot00000000000000; This test covers assertion "Typedef should have a parent scope" ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spt ; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc ; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM ; CHECK-SPIRV: String [[#TestStr:]] "test.cpp" ; CHECK-SPIRV: String [[#StdStr:]] "stddef.h" ; CHECK-SPIRV: [[#SourceCpp:]] [[#]] DebugSource [[#TestStr]] ; CHECK-SPIRV: [[#Scope:]] [[#]] DebugCompilationUnit [[#]] [[#]] [[#SourceCpp]] ; CHECK-SPIRV: [[#SourceHeader:]] [[#]] DebugSource [[#StdStr]] ; CHECK-SPIRV: DebugTypedef [[#]] [[#]] [[#SourceHeader]] [[#]] [[#]] [[#Scope]] ; CHECK-SPIRV: DebugEntryPoint [[#]] [[#Scope]] ; CHECK-LLVM: DW_TAG_typedef target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" $_ZTSZ4mainEUlvE_ = comdat any ; Function Attrs: convergent mustprogress norecurse nounwind define weak_odr dso_local spir_kernel void @_ZTSZ4mainEUlvE_() #0 comdat !dbg !14 { entry: ret void } attributes #0 = { convergent mustprogress norecurse nounwind } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11, !12, !13} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang based compiler", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3, imports: !2, splitDebugInlining: false, nameTableKind: None) !1 = !DIFile(filename: "test.cpp", directory: "") !2 = !{} !3 = !{!4} !4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression(DW_OP_constu, 175, DW_OP_stack_value)) !5 = distinct !DIGlobalVariable(name: "N", scope: !0, file: !1, line: 7, type: !6, isLocal: true, isDefinition: true) !6 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !7) !7 = !DIDerivedType(tag: DW_TAG_typedef, name: "size_t", file: !8, line: 62, baseType: !9) !8 = !DIFile(filename: "stddef.h", directory: "") !9 = !DIBasicType(name: "unsigned long", size: 64, encoding: DW_ATE_unsigned) !10 = !{i32 7, !"Dwarf Version", i32 4} !11 = !{i32 2, !"Debug Info Version", i32 3} !12 = !{i32 1, !"wchar_size", i32 4} !13 = !{i32 7, !"frame-pointer", i32 2} !14 = distinct !DISubprogram(name: "_ZTSZ4mainEUlvE_", scope: !1, file: !1, line: 24, type: !15, flags: DIFlagArtificial | DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !15 = !DISubroutineType(cc: DW_CC_nocall, types: !16) !16 = !{null} !17 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !18, entity: !19, file: !1, line: 58) !18 = !DINamespace(name: "std", scope: null) !19 = !DIDerivedType(tag: DW_TAG_typedef, name: "max_align_t", file: !1, baseType: !20) !20 = !DICompositeType(tag: DW_TAG_structure_type, file: !8, line: 19, size: 256, flags: DIFlagFwdDecl, elements: !2, identifier: "_ZTS11max_align_t") SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/FortranArray.ll000066400000000000000000000072041506646577400271500ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; Translation shouldn't crash: ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spt ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM ; CHECK-SPIRV: [[#CompUnit:]] [[#]] DebugCompilationUnit ; CHECK-SPIRV: [[#None:]] [[#]] DebugInfoNone ; CHECK-SPIRV: [[#EntryFunc:]] [[#]] DebugFunction ; CHECK-SPIRV: [[#BaseTy:]] [[#]] DebugTypeBasic ; CHECK-SPIRV: [[#Subrange:]] [[#]] DebugTypeSubrange ; CHECK-SPIRV: DebugTypeArrayDynamic [[#BaseTy]] [[#]] [[#]] [[#None]] [[#None]] [[#Subrange]] ; CHECK-SPIRV: DebugEntryPoint [[#EntryFunc]] [[#CompUnit]] [[#]] [[#]] {{$}} ; CHECK-LLVM: !DICompileUnit(language: DW_LANG_Fortran95 ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#BaseT:]], size: 32, elements: ![[#Elements:]], dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref, DW_OP_constu, 0, DW_OP_or)) ; CHECK-LLVM: ![[#BaseT:]] = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#Elements]] = !{![[#SubRange:]]} ; CHECK-LLVM: ![[#SubRange]] = !DISubrange(lowerBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref), upperBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref, DW_OP_push_object_address, DW_OP_plus_uconst, 48, DW_OP_deref, DW_OP_plus, DW_OP_constu, 1, DW_OP_minus), stride: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 56, DW_OP_deref)) source_filename = "llvm-link" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" !llvm.module.flags = !{!5, !6, !7, !8} !llvm.dbg.cu = !{!9} !0 = !{} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{i32 7, !"PIC Level", i32 2} !7 = !{i32 2, !"Debug Info Version", i32 3} !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !10, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !0, globals: !0, imports: !22, splitDebugInlining: false, nameTableKind: None) !10 = !DIFile(filename: "declare_target_subroutine.F90", directory: "/test") !19 = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) !22 = !{!23} !23 = !DIImportedEntity(tag: DW_TAG_imported_module, scope: !24, entity: !34, file: !10, line: 24) !24 = distinct !DISubprogram(name: "declare_target_subroutine", linkageName: "MAIN__", scope: !10, file: !10, line: 23, type: !25, scopeLine: 23, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !9, retainedNodes: !27) !25 = !DISubroutineType(types: !26) !26 = !{null} !27 = !{!30} !30 = !DILocalVariable(name: "a", scope: !24, file: !10, line: 28, type: !31) !31 = !DICompositeType(tag: DW_TAG_array_type, baseType: !19, elements: !32, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref, DW_OP_constu, 0, DW_OP_or)) !32 = !{!33} !33 = !DISubrange(lowerBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref), upperBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref, DW_OP_push_object_address, DW_OP_plus_uconst, 48, DW_OP_deref, DW_OP_plus, DW_OP_constu, 1, DW_OP_minus), stride: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 56, DW_OP_deref)) !34 = !DIModule(scope: !24, name: "iso_fortran_env", isDecl: true) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/FortranArrayNoType.spt000066400000000000000000000043111506646577400305020ustar00rootroot00000000000000; Tests translation of DebugTypeArrayDynamic with DebugInfoNone type ; RUN: llvm-spirv -to-binary %s -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.bc ; RUN: llvm-dis < %t.bc | FileCheck %s --check-prefix=CHECK-LLVM ; CHECK-LLVM: ![[#]] = !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#Type:]] ; CHECK-LLVM: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "SPIRV unknown type") 119734787 65536 393230 49 0 2 Capability Addresses 2 Capability Linkage 2 Capability Kernel 8 Extension "SPV_KHR_non_semantic_info" 5 ExtInstImport 1 "OpenCL.std" 11 ExtInstImport 2 "NonSemantic.Shader.DebugInfo.200" 3 MemoryModel 2 2 11 String 7 "/test/declare_target_subroutine.F90" 3 String 9 "0" 3 String 13 "" 9 String 22 "declare_target_subroutine" 4 String 23 "MAIN__" 3 String 27 "a" 5 String 28 "INTEGER*4" 6 String 42 "iso_fortran_env" 3 Source 0 0 4 Name 5 "MAIN__" 6 Decorate 5 LinkageAttributes "MAIN__" Export 4 TypeInt 10 32 0 4 Constant 10 11 1 4 Constant 10 15 65536 4 Constant 10 16 4 4 Constant 10 17 206 4 Constant 10 20 0 4 Constant 10 24 23 4 Constant 10 25 8 4 Constant 10 29 32 4 Constant 10 31 149 4 Constant 10 40 28 4 Constant 10 45 24 2 TypeVoid 3 3 TypeFunction 4 3 6 ExtInst 3 8 2 DebugSource 7 7 ExtInst 3 12 2 DebugBuildIdentifier 9 11 6 ExtInst 3 14 2 DebugStoragePath 13 10 ExtInst 3 18 2 DebugCompilationUnit 15 16 8 17 13 5 ExtInst 3 19 2 DebugInfoNone 7 ExtInst 3 21 2 DebugTypeFunction 20 19 15 ExtInst 3 26 2 DebugFunction 22 21 8 24 20 18 23 25 24 19 5 ExtInst 3 30 2 DebugInfoNone 6 ExtInst 3 32 2 DebugOperation 31 6 ExtInst 3 33 2 DebugExpression 32 6 ExtInst 3 34 2 DebugOperation 31 6 ExtInst 3 35 2 DebugExpression 34 6 ExtInst 3 36 2 DebugOperation 31 6 ExtInst 3 37 2 DebugExpression 36 9 ExtInst 3 38 2 DebugTypeSubrange 37 37 19 37 11 ExtInst 3 39 2 DebugTypeArrayDynamic 30 33 35 19 19 38 12 ExtInst 3 41 2 DebugLocalVariable 27 39 8 40 20 26 20 6 ExtInst 3 43 2 DebugSource 13 13 ExtInst 3 44 2 DebugModule 42 43 20 26 13 13 13 11 12 ExtInst 3 46 2 DebugImportedEntity 13 20 8 44 45 20 26 9 ExtInst 3 47 2 DebugEntryPoint 26 18 13 13 5 Function 3 5 0 4 2 Label 6 7 ExtInst 3 48 2 DebugFunctionDefinition 26 5 1 Return 1 FunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/FortranComplex.ll000066400000000000000000000141621506646577400275020ustar00rootroot00000000000000;; Tests for Fortran's complex type encoding in debug info ;; Compiled from the following Fortran source ;; ;; program complex_numbers ;; implicit none ;; complex :: a, b, c ;; ;; a = (1.0, 2.0) ;; b = (2.0, -1.0) ;; ;; c = a + b ;; ;; end program complex_numbers ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o - | FileCheck %s --check-prefix=CHECK-SPIRV-200 ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM-200 ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-100 -o - | FileCheck %s --check-prefix=CHECK-SPIRV-100 ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-100 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM-100 ; CHECK-SPIRV-200-DAG: ExtInstImport [[#Import:]] "NonSemantic.Shader.DebugInfo.200 ; CHECK-SPIRV-200-DAG: String [[#Name:]] "COMPLEX*8" ; CHECK-SPIRV-200-DAG: Constant [[#]] [[#Size:]] 64 ; CHECK-SPIRV-200-DAG: Constant [[#]] [[#Encoding:]] 8 ; CHECK-SPIRV-200-DAG: ExtInst [[#]] [[#Type:]] [[#Import]] DebugTypeBasic [[#Name]] [[#Size]] [[#Encoding]] ; CHECK-SPIRV-200-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-SPIRV-200-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-SPIRV-200-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-LLVM-200-DAG: ![[#]] = !DILocalVariable(name: "a", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type:]]) ; CHECK-LLVM-200-DAG: ![[#Type]] = !DIBasicType(name: "COMPLEX*8", size: 64, encoding: DW_ATE_complex_float) ; CHECK-LLVM-200-DAG: ![[#]] = !DILocalVariable(name: "b", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type]]) ; CHECK-LLVM-200-DAG: ![[#]] = !DILocalVariable(name: "c", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type]]) ; CHECK-SPIRV-100-DAG: ExtInstImport [[#Import:]] "NonSemantic.Shader.DebugInfo.100 ; CHECK-SPIRV-100-DAG: String [[#Name:]] "COMPLEX*8" ; CHECK-SPIRV-100-DAG: Constant [[#]] [[#Size:]] 64 ; CHECK-SPIRV-100-DAG: Constant [[#]] [[#Encoding:]] 0 ; CHECK-SPIRV-100-DAG: ExtInst [[#]] [[#Type:]] [[#Import]] DebugTypeBasic [[#Name]] [[#Size]] [[#Encoding]] ; CHECK-SPIRV-100-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-SPIRV-100-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-SPIRV-100-DAG: ExtInst [[#]] [[#]] [[#Import]] DebugLocalVariable [[#]] [[#Type]] ; CHECK-LLVM-100-DAG: ![[#]] = !DILocalVariable(name: "a", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type:]]) ; CHECK-LLVM-100-DAG: ![[#Type]] = !DIBasicType(tag: DW_TAG_unspecified_type, name: "COMPLEX*8") ; CHECK-LLVM-100-DAG: ![[#]] = !DILocalVariable(name: "b", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type]]) ; CHECK-LLVM-100-DAG: ![[#]] = !DILocalVariable(name: "c", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#Type]]) ; ModuleID = 'test.f90' source_filename = "test.f90" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" @0 = internal unnamed_addr constant i32 65536, align 4 @1 = internal unnamed_addr constant i32 2, align 4 ; Function Attrs: nounwind uwtable define void @MAIN__() local_unnamed_addr !dbg !4 !llfort.type_idx !12 { alloca_0: %func_result = tail call i32 @for_set_fpe_(ptr nonnull @0), !dbg !13, !llfort.type_idx !14 %func_result2 = tail call i32 @for_set_reentrancy(ptr nonnull @1), !dbg !13, !llfort.type_idx !14 call void @llvm.dbg.value(metadata float 1.000000e+00, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !15 call void @llvm.dbg.value(metadata float 2.000000e+00, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 32, 32)), !dbg !15 call void @llvm.dbg.value(metadata float 2.000000e+00, metadata !10, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !15 call void @llvm.dbg.value(metadata float -1.000000e+00, metadata !10, metadata !DIExpression(DW_OP_LLVM_fragment, 32, 32)), !dbg !15 call void @llvm.dbg.value(metadata float poison, metadata !8, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !15 call void @llvm.dbg.value(metadata float poison, metadata !8, metadata !DIExpression(DW_OP_LLVM_fragment, 32, 32)), !dbg !15 ret void, !dbg !16 } declare !llfort.intrin_id !17 !llfort.type_idx !18 i32 @for_set_fpe_(ptr nocapture readonly) local_unnamed_addr ; Function Attrs: nofree declare !llfort.intrin_id !19 !llfort.type_idx !20 i32 @for_set_reentrancy(ptr nocapture readonly) local_unnamed_addr ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) declare void @llvm.dbg.value(metadata, metadata, metadata) !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !omp_offload.info = !{} !0 = !{i32 2, !"Debug Info Version", i32 3} !1 = !{i32 2, !"Dwarf Version", i32 4} !2 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !3, producer: "Intel(R) Fortran", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) !3 = !DIFile(filename: "test.f90", directory: "complex") !4 = distinct !DISubprogram(name: "complex_numbers", linkageName: "MAIN__", scope: !3, file: !3, line: 1, type: !5, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2, retainedNodes: !7) !5 = !DISubroutineType(types: !6) !6 = !{null} !7 = !{!8, !10, !11} !8 = !DILocalVariable(name: "c", scope: !4, file: !3, line: 3, type: !9) !9 = !DIBasicType(name: "COMPLEX*8", size: 64, encoding: DW_ATE_complex_float) !10 = !DILocalVariable(name: "b", scope: !4, file: !3, line: 3, type: !9) !11 = !DILocalVariable(name: "a", scope: !4, file: !3, line: 3, type: !9) !12 = !{i64 23} !13 = !DILocation(line: 1, column: 9, scope: !4) !14 = !{i64 2} !15 = !DILocation(line: 0, scope: !4) !16 = !DILocation(line: 9, column: 1, scope: !4) !17 = !{i32 97} !18 = !{i64 27} !19 = !{i32 98} !20 = !{i64 29} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/FortranDynamicArrayExpr.ll000066400000000000000000000134171506646577400313170ustar00rootroot00000000000000;; The test checks, that Fortran dynamic arrays are being correctly represented ;; by SPIR-V debug information ;; Unlike 'static' arrays dynamic can have following parameters of ;; DICompositeType metadata with DW_TAG_array_type tag: ;; Data Location, Associated, Allocated and Rank which can be represented ;; by either DIExpression or DIVariable (both local and global). ;; This test if for expression representation. ;; FortranDynamicArrayVar.ll is for variable representation. ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM ; CHECK-SPIRV-DAG: ExtInstImport [[#Import:]] "NonSemantic.Shader.DebugInfo.200" ; CHECK-SPIRV-DAG: String [[#BasicTName:]] "INTEGER*4" ; CHECK-SPIRV-DAG: TypeInt [[#Int32T:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#Int32T]] [[#IntConst:]] 32 ; CHECK-SPIRV-DAG: Constant [[#Int32T]] [[#Flag:]] 4 ; CHECK-SPIRV-DAG: TypeVoid [[#VoidT:]] ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgInfoNone:]] [[#Import]] DebugInfoNone ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#ArrayBasicT:]] [[#Import]] DebugTypeBasic [[#BasicTName]] [[#IntConst]] [[#Flag]] ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgExprLocation:]] [[#Import]] DebugExpression [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgExprAssociated:]] [[#Import]] DebugExpression [[#]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgExprLowerBound:]] [[#Import]] DebugExpression [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgExprUpperBound:]] [[#Import]] DebugExpression [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgExprStride:]] [[#Import]] DebugExpression [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgSubRangeId:]] [[#Import]] DebugTypeSubrange [[#DbgExprLowerBound]] [[#DbgExprUpperBound]] [[#DbgInfoNone]] [[#DbgExprStride]] ; CHECK-SPIRV: ExtInst [[#VoidT]] [[#DbgArrayId:]] [[#Import]] DebugTypeArrayDynamic [[#ArrayBasicT]] [[#DbgExprLocation]] [[#DbgExprAssociated]] [[#DbgInfoNone]] [[#DbgInfoNone]] [[#DbgSubRangeId]] ; CHECK-LLVM: %[[#Array:]] = alloca ; CHECK-LLVM: call void @llvm.dbg.value(metadata ptr %[[#Array]], metadata ![[#DbgLVar:]] ; CHECK-LLVM: ![[#DbgLVar]] = !DILocalVariable(name: "pint", scope: ![[#]], file: ![[#]], line: [[#]], type: ![[#DbgLVarT:]]) ; CHECK-LLVM: ![[#DbgLVarT]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[#DbgArrayT:]], size: 64) ; CHECK-LLVM: ![[#DbgArrayT]] = !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#DbgArrayBaseT:]], size: 32, elements: ![[#Elements:]], dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref, DW_OP_constu, 0, DW_OP_or)) ; CHECK-LLVM: ![[#DbgArrayBaseT]] = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#Elements]] = !{![[#SubRange:]]} ; CHECK-LLVM: ![[#SubRange]] = !DISubrange(lowerBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref), upperBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref, DW_OP_push_object_address, DW_OP_plus_uconst, 48, DW_OP_deref, DW_OP_plus, DW_OP_constu, 1, DW_OP_minus), stride: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 56, DW_OP_deref)) ; ModuleID = 'reproducer.ll' source_filename = "test.f90" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" %qnca = type { i32 addrspace(4)*, i64, i64, i64, i64, i64, [1 x { i64, i64, i64 }] } ; Function Attrs: noinline nounwind optnone define weak dso_local spir_kernel void @TEST() #0 !dbg !5 { newFuncRoot: %0 = alloca %qnca, align 8 call void @llvm.dbg.value(metadata %qnca* %0, metadata !8, metadata !DIExpression()), !dbg !14 ret void } ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { noinline nounwind optnone } attributes #1 = { nofree nosync nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !spirv.Source = !{!4} !0 = !{i32 2, !"Debug Info Version", i32 3} !1 = !{i32 2, !"Dwarf Version", i32 4} !2 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !3, producer: "fortran", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) !3 = !DIFile(filename: "test.f90", directory: "/path/to") !4 = !{i32 4, i32 200000} !5 = distinct !DISubprogram(name: "test", linkageName: "MAIN__", scope: !3, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2) !6 = !DISubroutineType(types: !7) !7 = !{null} !8 = !DILocalVariable(name: "pint", scope: !5, file: !3, line: 3, type: !9) !9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64) !10 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, elements: !12, dataLocation: !DIExpression(DW_OP_push_object_address, DW_OP_deref), associated: !DIExpression(DW_OP_push_object_address, DW_OP_deref, DW_OP_constu, 0, DW_OP_or)) !11 = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) !12 = !{!13} !13 = !DISubrange(lowerBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref), upperBound: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 64, DW_OP_deref, DW_OP_push_object_address, DW_OP_plus_uconst, 48, DW_OP_deref, DW_OP_plus, DW_OP_constu, 1, DW_OP_minus), stride: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 56, DW_OP_deref)) !14 = !DILocation(line: 1, scope: !5) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/FortranDynamicArrayVar.ll000066400000000000000000000103531506646577400311250ustar00rootroot00000000000000;; DebugInfo/dwarfdump-dataLocationVar.ll from llvm.org is used as base for this test ;; The test checks, that Fortran dynamic arrays are being correctly represented ;; by SPIR-V debug information ;; Unlike 'static' arrays dynamic can have following parameters of ;; DICompositeType metadata with DW_TAG_array_type tag: ;; Data Location, Associated, Allocated and Rank which can be represented ;; by either DIExpression or DIVariable (both local and global). ;; This test if for variable representation. ;; FortranDynamicArrayVar.ll is for expression representation. ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text --spirv-debug-info-version=nonsemantic-shader-200 -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefix=CHECK-LLVM ;; Major SPIR-V checks are done in FortranDynamicArrayExpr.ll ; CHECK-SPIRV: ExtInst [[#]] [[#DbgLVarId:]] [[#]] DebugLocalVariable [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugTypeArrayDynamic [[#]] [[#DbgLVarId]] [[#]] [[#]] [[#]] [[#]] ; CHECK-LLVM: %[[#Ptr:]] = alloca ptr ; CHECK-LLVM: %[[#Array:]] = alloca [16 x i64] ; CHECK-LLVM: call void @llvm.dbg.declare(metadata ptr %[[#Array]], metadata ![[#DbgLVarArray:]] ; CHECK-LLVM: call void @llvm.dbg.declare(metadata ptr %[[#Ptr]], metadata ![[#DbgLVarPtr:]] ; CHECK-LLVM: ![[#DbgLVarPtr:]] = !DILocalVariable(scope: ![[#]], file: ![[#]], type: ![[#DbgPtrT:]], flags: DIFlagArtificial) ; CHECK-LLVM: ![[#DbgPtrT:]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[#DbgBasicT:]], size: 64) ; CHECK-LLVM: ![[#DbgBasicT]] = !DIBasicType(name: "integer", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#DbgLVarArray]] = !DILocalVariable(name: "arr", scope: ![[#]], file: ![[#]], type: ![[#DbgArrayT:]]) ; CHECK-LLVM: ![[#DbgArrayT]] = !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#DbgBasicT]], size: 608, elements: ![[#Elements:]], dataLocation: ![[#DbgLVarPtr]]) ; CHECK-LLVM: ![[#Elements]] = !{![[#SubRange:]]} ; CHECK-LLVM: ![[#SubRange]] = !DISubrange(count: 19, lowerBound: 2) ; ModuleID = 'fortsubrange.ll' source_filename = "fortsubrange.ll" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" define spir_func void @foo() !dbg !5 { L.entry: %0 = alloca ptr, align 8 %1 = alloca [16 x i64], align 8 call void @llvm.dbg.declare(metadata ptr %1, metadata !8, metadata !DIExpression()), !dbg !16 call void @llvm.dbg.declare(metadata ptr %0, metadata !14, metadata !DIExpression()), !dbg !16 ret void, !dbg !17 } ; Function Attrs: nounwind readnone speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) ; Function Attrs: nounwind readnone speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !0 = !{i32 2, !"Dwarf Version", i32 4} !1 = !{i32 2, !"Debug Info Version", i32 3} !2 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !3, producer: " F95 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4) !3 = !DIFile(filename: "fortsubrange.f90", directory: "/dir") !4 = !{} !5 = distinct !DISubprogram(name: "main", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2) !6 = !DISubroutineType(cc: DW_CC_program, types: !7) !7 = !{null} !8 = !DILocalVariable(name: "arr", scope: !9, file: !3, type: !10) !9 = !DILexicalBlock(scope: !5, file: !3, line: 1, column: 1) !10 = !DICompositeType(tag: DW_TAG_array_type, baseType: !11, size: 32, align: 32, elements: !12, dataLocation: !14) !11 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed) !12 = !{!13} !13 = !DISubrange(count: 19, lowerBound: 2) !14 = distinct !DILocalVariable(scope: !9, file: !3, type: !15, flags: DIFlagArtificial) !15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 32, align: 32) !16 = !DILocation(line: 0, scope: !9) !17 = !DILocation(line: 6, column: 1, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/InlineNamespace.ll000077500000000000000000000062001506646577400275670ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o %t.spt --spirv-debug-info-version=nonsemantic-shader-200 ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -to-binary %t.spt -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; Source: ; -- inline-namespace.cxx ----------------------------------------------------- ; 1 inline namespace Outer { ; 2 namespace Inner { ; 3 int global; ; 4 } ; 5 } ; 6 ; 7 void foo() { ; 8 Inner::global++; ; 9 } ; -- inline-namespace.cxx ----------------------------------------------------- ; CHECK-SPIRV: String [[#StrOuter:]] "Outer" ; CHECK-SPIRV: String [[#StrInner:]] "Inner" ; CHECK-SPIRV: TypeBool [[#TypeBool:]] ; CHECK-SPIRV: ConstantTrue [[#TypeBool]] [[#ConstTrue:]] ; CHECK-SPIRV: ConstantFalse [[#TypeBool]] [[#ConstFalse:]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugLexicalBlock [[#]] [[#]] [[#]] [[#]] [[#StrOuter]] [[#ConstTrue]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugLexicalBlock [[#]] [[#]] [[#]] [[#]] [[#StrInner]] [[#ConstFalse]] ; CHECK-LLVM: !DINamespace(name: "Inner", scope: ![[#OuterSpace:]]) ; CHECK-LLVM: ![[#OuterSpace]] = !DINamespace(name: "Outer", scope: null, exportSymbols: true) ; ModuleID = 'inline-namespace.cxx' target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" @_ZN5Outer5Inner6globalE = dso_local global i32 0, align 4, !dbg !0 ; Function Attrs: noinline nounwind optnone uwtable mustprogress define dso_local void @_Z3foov() #0 !dbg !11 { entry: %0 = load i32, i32* @_ZN5Outer5Inner6globalE, align 4, !dbg !14 %inc = add nsw i32 %0, 1, !dbg !14 store i32 %inc, i32* @_ZN5Outer5Inner6globalE, align 4, !dbg !14 ret void, !dbg !15 } attributes #0 = { noinline nounwind optnone uwtable mustprogress } !llvm.dbg.cu = !{!6} !llvm.module.flags = !{!9, !10} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "global", linkageName: "_ZN5Outer5Inner6globalE", scope: !2, file: !4, line: 3, type: !5, isLocal: false, isDefinition: true) !2 = !DINamespace(name: "Inner", scope: !3) !3 = !DINamespace(name: "Outer", scope: null, exportSymbols: true) !4 = !DIFile(filename: "inlined-namespace.cxx", directory: "/path/to") !5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !4, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !7, globals: !8, splitDebugInlining: false, nameTableKind: None) !7 = !{} !8 = !{!0} !9 = !{i32 7, !"Dwarf Version", i32 4} !10 = !{i32 2, !"Debug Info Version", i32 3} !11 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !4, file: !4, line: 7, type: !12, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !7) !12 = !DISubroutineType(types: !13) !13 = !{null} !14 = !DILocation(line: 8, column: 16, scope: !11) !15 = !DILocation(line: 9, column: 1, scope: !11) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/SourceLanguageLLVMToSPIRV.ll000066400000000000000000000050041506646577400312600ustar00rootroot00000000000000; Test checks that DW_LANG_C99, DW_LANG_OpenCL, and all DW_LANG_C_plus_plus_X are mapped to ; appropriate SourceLanguages in SPIRV when the extended debug info is enabled. ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C99/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-C99 ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_OpenCL/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-OPENCLC ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C_plus_plus/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-CPP ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C_plus_plus_14/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-CPP14 ; CHECK-C99: Constant [[#]] [[#Constant:]] 211 ; CHECK-C99: DebugCompilationUnit [[#]] [[#]] [[#]] [[#Constant]] ; CHECK-OPENCLC: Constant [[#]] [[#Constant:]] 3 ; CHECK-OPENCLC: DebugCompilationUnit [[#]] [[#]] [[#]] [[#Constant]] ; CHECK-CPP: Constant [[#]] [[#Constant:]] 214 ; CHECK-CPP: DebugCompilationUnit [[#]] [[#]] [[#]] [[#Constant]] ; CHECK-CPP14: Constant [[#]] [[#Constant:]] 217 ; CHECK-CPP14: DebugCompilationUnit [[#]] [[#]] [[#]] [[#Constant]] target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64-unknown-unknown" define dso_local spir_kernel void @func() local_unnamed_addr !dbg !7 !kernel_arg_addr_space !2 !kernel_arg_access_qual !2 !kernel_arg_type !2 !kernel_arg_base_type !2 !kernel_arg_type_qual !2 { entry: ret void } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !0 = distinct !DICompileUnit(language: INPUT_LANGUAGE, file: !1) !1 = !DIFile(filename: "test.cl", directory: "/tmp", checksumkind: CSK_MD5, checksum: "18aa9ce738eaafc7b7b7181c19092815") !2 = !{} !3 = !{i32 7, !"Dwarf Version", i32 5} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{i32 2, i32 0} !7 = distinct !DISubprogram(name: "func", scope: !8, file: !8, line: 1, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !8 = !DIFile(filename: "test.cl", directory: "/tmp", checksumkind: CSK_MD5, checksum: "18aa9ce738eaafc7b7b7181c19092815") SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/Shader200/storagePath_dwo.ll000066400000000000000000000033251506646577400276700ustar00rootroot00000000000000; Test checks that dwoId and splitDebugFilename is preserved from LLVM IR to spirv ; and spirv to LLVM IR translation. ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV ; RUN: llvm-spirv %t.bc --spirv-debug-info-version=nonsemantic-shader-200 -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck %s --input-file %t.rev.ll --check-prefix CHECK-LLVM ; CHECK-SPIRV: String [[#stringA_id:]] "11111" ; CHECK-SPIRV: String [[#stringA_sf:]] "debugA_info.dwo" ; CHECK-SPIRV: [[#buildID_A:]] [[#]] DebugBuildIdentifier [[#stringA_id]] ; CHECK-SPIRV: [[#storageID_A:]] [[#]] DebugStoragePath [[#stringA_sf]] ; CHECK-LLVM: !DICompileUnit ; CHECK-LLVM-SAME: splitDebugFilename: "debugA_info.dwo" ; CHECK-LLVM-SAME: dwoId: 11111 ; CHECK-LLVM: !DICompileUnit ; CHECK-LLVM-SAME: splitDebugFilename: "debugA_info.dwo" ; CHECK-LLVM-SAME: dwoId: 11111 !llvm.dbg.cu = !{!7, !0} !llvm.module.flags = !{!3, !4} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Clang", isOptimized: false, runtimeVersion: 2, splitDebugFilename: "debugA_info.dwo", emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2, dwoId: 11111) !1 = !DIFile(filename: "", directory: "/") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{!6} !6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !7 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Clang", isOptimized: false, runtimeVersion: 2, splitDebugFilename: "debugA_info.dwo", dwoId: 11111, emissionKind: FullDebug, retainedTypes: !5) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/basic-type-with-flag.spt000066400000000000000000000047171506646577400272330ustar00rootroot00000000000000;; This test checks, that SPIR-V module generated by another SPIR-V producer ;; containing FlagUnknownPhysicalLayout flag of DebugTypeBasic can be ;; consumed without issues ; RUN: llvm-spirv %s -to-binary -o %t.spv ; RUN: llvm-spirv -r %t.spv ; RUN: llvm-dis < %t.bc | FileCheck %s --check-prefix=CHECK-LLVM ; CHECK-LLVM: DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) 119734787 65792 393230 56 0 2 Capability Addresses 2 Capability Linkage 2 Capability Kernel 2 Capability Int8 8 Extension "SPV_KHR_non_semantic_info" 5 ExtInstImport 1 "OpenCL.std" 11 ExtInstImport 2 "NonSemantic.Shader.DebugInfo.100" 3 MemoryModel 2 2 8 String 25 "foo.cpp" 4 String 30 "int" 3 String 35 "A" 3 String 36 "b" 3 String 44 "foo" 5 String 45 "_Z3fooP1A" 3 String 51 "a" 3 Source 0 0 5 Name 7 "_Z3fooP1A" 3 Name 8 "a" 4 Name 9 "entry" 4 Name 11 "a.addr" 5 Name 16 "struct.A" 3 Name 23 "b" 15 ModuleProcessed "Debug info producer: clang version 3.1 (trunk 150996)" 7 Decorate 7 LinkageAttributes "_Z3fooP1A" Export 4 Decorate 11 Alignment 8 4 TypeInt 3 32 0 4 TypeInt 4 8 0 4 Constant 3 21 0 4 Constant 3 27 65536 4 Constant 3 28 6 4 Constant 3 31 32 4 Constant 3 32 4 4 Constant 3 37 1 4 Constant 3 38 2 4 Constant 3 41 4294967295 4 Constant 3 46 3 4 Constant 3 47 136 4 Constant 3 49 16 4 Constant 3 56 131072 ;; FlagUnknownPhysicalLayout 4 TypePointer 5 7 4 4 TypeFunction 6 3 5 4 TypePointer 10 7 5 2 TypeVoid 13 3 TypeStruct 16 3 4 TypePointer 17 7 16 4 TypePointer 18 7 17 4 TypePointer 22 7 3 5 ExtInst 13 14 2 DebugInfoNone 7 ExtInst 13 26 2 DebugSource 25 14 9 ExtInst 13 29 2 DebugCompilationUnit 27 21 26 28 9 ExtInst 13 33 2 DebugTypeBasic 30 31 32 56 14 ExtInst 13 39 2 DebugTypeMember 36 33 26 37 21 34 21 31 38 15 ExtInst 13 34 2 DebugTypeComposite 35 21 26 37 21 29 14 31 21 39 8 ExtInst 13 42 2 DebugTypePointer 34 41 21 8 ExtInst 13 43 2 DebugTypeFunction 21 33 42 15 ExtInst 13 48 2 DebugFunction 44 43 26 46 21 29 45 47 46 14 9 ExtInst 13 50 2 DebugLexicalBlock 26 46 49 48 13 ExtInst 13 52 2 DebugLocalVariable 51 42 26 46 21 48 21 1 5 ExtInst 13 53 2 DebugExpression 5 Function 3 7 0 6 3 FunctionParameter 5 8 2 Label 9 7 ExtInst 13 57 2 DebugFunctionDefinition 48 7 4 Variable 10 11 7 4 Bitcast 10 12 11 5 Store 12 8 2 8 6 ExtInst 13 54 2 DebugScope 48 4 Line 25 3 13 8 ExtInst 13 15 2 DebugDeclare 52 11 53 6 ExtInst 13 55 2 DebugScope 50 4 Line 25 4 3 4 Bitcast 18 19 11 6 Load 17 20 19 2 8 6 InBoundsPtrAccessChain 22 23 20 21 21 6 Load 3 24 23 2 4 2 ReturnValue 24 1 FunctionEnd SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/NonSemantic/static_member_array.ll000066400000000000000000000126161506646577400271250ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv -spirv-debug-info-version=nonsemantic-shader-100 ; RUN: llvm-spirv %t.spv --to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; RUN: llvm-spirv %t.bc -o %t.spv -spirv-debug-info-version=nonsemantic-shader-200 ; RUN: llvm-spirv %t.spv --to-text -o - | FileCheck %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; Generated from: ; ; struct A { ; static int fully_specified; ; static int smem[]; ; }; ; ; int A::fully_specified; ; int A::smem[] = { 0, 1, 2, 3 }; ; CHECK-SPIRV: ExtInst [[#]] [[#Member1:]] [[#]] DebugTypeMember [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#]] [[#Member2:]] [[#]] DebugTypeMember [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] {{$}} ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugTypeComposite [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#Member1]] [[#Member2]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugGlobalVariable [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#Member1]] ; CHECK-SPIRV: ExtInst [[#]] [[#]] [[#]] DebugGlobalVariable [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#]] [[#Member2]] ; CHECK-LLVM: ![[#GVExpr1:]] = !DIGlobalVariableExpression(var: ![[#GV1:]], expr: !DIExpression()) ; CHECK-LLVM: ![[#GV1]] = distinct !DIGlobalVariable(name: "fully_specified", linkageName: "_ZN1A15fully_specifiedE", scope: ![[#CU:]], file: ![[#File:]], line: 7, type: ![[#GVTy1:]], isLocal: false, isDefinition: true, declaration: ![[#Decl1:]]) ; CHECK-LLVM: ![[#CU]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus{{.*}}, file: ![[#File:]], {{.*}}, globals: ![[#GVs:]] ; CHECK-LLVM: ![[#File]] = !DIFile(filename: "static_member_array.cpp", directory: "/Volumes/Data/radar/28706946") ; CHECK-LLVM: ![[#GVs]] = !{![[#GVExpr1]], ![[#GVExpr2:]]} ; CHECK-LLVM: ![[#GVExpr2]] = !DIGlobalVariableExpression(var: ![[#GV2:]], expr: !DIExpression()) ; CHECK-LLVM: ![[#GV2]] = distinct !DIGlobalVariable(name: "smem", linkageName: "_ZN1A4smemE", scope: ![[#CU]], file: ![[#File]], line: 8, type: ![[#GVTy2:]], isLocal: false, isDefinition: true, declaration: ![[#Decl2:]]) ; CHECK-LLVM: ![[#GVTy2]] = !DICompositeType(tag: DW_TAG_array_type, baseType: !8, size: 128, elements: ![[#Elements1:]]) ; CHECK-LLVM: ![[#GVTy1]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#Elements1]] = !{![[#Subrange:]]} ; CHECK-LLVM: ![[#Subrange]] = !DISubrange(count: 4 ; CHECK-LLVM: ![[#MemTy1:]] = !DIDerivedType(tag: DW_TAG_member, name: "smem", scope: ![[#StructTy:]], file: ![[#File]], line: 4, baseType: ![[#ArrTy:]], flags: DIFlagPublic | DIFlagStaticMember) ; CHECK-LLVM: ![[#StructTy]] = !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: ![[#File]], line: 1, size: 8, elements: ![[#Elements2:]], identifier: "_ZTS1A") ; CHECK-LLVM: ![[#Elements2]] = !{![[#MemTy2:]], ![[#MemTy1]]} ; CHECK-LLVM: ![[#MemTy2:]] = !DIDerivedType(tag: DW_TAG_member, name: "fully_specified", scope: ![[#StructTy]], file: ![[#File]], line: 3, baseType: ![[#GVTy1]], flags: DIFlagPublic | DIFlagStaticMember) ; CHECK-LLVM: ![[#ArrTy]] = !DICompositeType(tag: DW_TAG_array_type, baseType: !8, elements: ![[#]]) source_filename = "static_member_array.cpp" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" @_ZN1A15fully_specifiedE = global i32 0, align 4, !dbg !0 @_ZN1A4smemE = global [4 x i32] [i32 0, i32 1, i32 2, i32 3], align 16, !dbg !6 !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!19, !20, !21} !llvm.ident = !{!22} !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "fully_specified", linkageName: "_ZN1A15fully_specifiedE", scope: !2, file: !3, line: 7, type: !9, isLocal: false, isDefinition: true, declaration: !15) !2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 4.0.0 (trunk 286129) (llvm/trunk 286128)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5) !3 = !DIFile(filename: "static_member_array.cpp", directory: "/Volumes/Data/radar/28706946") !4 = !{} !5 = !{!0, !6} !6 = distinct !DIGlobalVariableExpression(var: !7, expr: !DIExpression()) !7 = !DIGlobalVariable(name: "smem", linkageName: "_ZN1A4smemE", scope: !2, file: !3, line: 8, type: !8, isLocal: false, isDefinition: true, declaration: !12) !8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 128, elements: !10) !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !10 = !{!11} !11 = !DISubrange(count: 4) !12 = !DIDerivedType(tag: DW_TAG_member, name: "smem", scope: !13, file: !3, line: 4, baseType: !16, flags: DIFlagStaticMember) !13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !3, line: 1, size: 8, elements: !14, identifier: "_ZTS1A") !14 = !{!15, !12} !15 = !DIDerivedType(tag: DW_TAG_member, name: "fully_specified", scope: !13, file: !3, line: 3, baseType: !9, flags: DIFlagStaticMember) !16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, elements: !17) !17 = !{!18} !18 = !DISubrange(count: -1) !19 = !{i32 2, !"Dwarf Version", i32 4} !20 = !{i32 2, !"Debug Info Version", i32 3} !21 = !{i32 1, !"PIC Level", i32 2} !22 = !{!"clang version 4.0.0 (trunk 286129) (llvm/trunk 286128)"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/OpenCL100/000077500000000000000000000000001506646577400216755ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/OpenCL100/DebugInfoSubrange.ll000066400000000000000000000200161506646577400255560ustar00rootroot00000000000000; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o %t.spt ; RUN: FileCheck < %t.spt %s -check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -to-binary %t.spt -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc -o %t.rev.ll ; RUN: FileCheck < %t.rev.ll %s -check-prefix=CHECK-LLVM ; CHECK-SPIRV: String [[#VarNameId:]] "A$1$upperbound" ; CHECK-SPIRV: [[#FuncNameId:]] "random_fill_sp" ; CHECK-SPIRV: TypeInt [[#TypeInt64Id:]] 64 0 ; CHECK-SPIRV: Constant [[#TypeInt64Id]] [[#LowerBoundId:]] 1 0 ; CHECK-SPIRV: Constant [[#TypeInt64Id]] [[#NegativeCount:]] 4294967295 4294967295 ; CHECK-SPIRV: [[#DbgFuncId:]] [[#]] DebugFunction [[#FuncNameId]] ; CHECK-SPIRV: [[#]] [[#DbgLocVarId:]] [[#]] DebugLocalVariable [[#VarNameId]] [[#]] [[#]] [[#]] [[#]] [[#DbgFuncId]] ; CHECK-SPIRV: [[#DbgTemplateId:]] [[#]] DebugTypeTemplate [[#DbgFuncId]] ; CHECK-SPIRV: DebugTypeArray [[#]] [[#DbgLocVarId]] [[#LowerBoundId]] ; CHECK-SPIRV: [[#DbgExprId:]] [[#]] DebugExpression ; CHECK-SPIRV: DebugTypeArray [[#]] [[#DbgExprId]] [[#DbgExprId]] ; CHECK-SPIRV: DebugTypeArray [[#]] [[#NegativeCount]] [[#]] ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#BaseType:]], size: 32, elements: ![[#Subrange1:]]) ; CHECK-LLVM: [[#BaseType]] = !DIBasicType(name: "REAL*4", size: 32, encoding: DW_ATE_float) ; CHECK-LLVM: [[#Subrange1]] = !{![[#Subrange2:]]} ; CHECK-LLVM: [[#Subrange2:]] = !DISubrange(lowerBound: 1, upperBound: ![[#UpperBound:]]) ; CHECK-LLVM: [[#UpperBound]] = !DILocalVariable(name: "A$1$upperbound" ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#]], size: 32, elements: ![[#SubrangeExpr1:]]) ; CHECK-LLVM: [[#SubrangeExpr1]] = !{![[#SubrangeExpr2:]]} ; CHECK-LLVM: ![[#SubrangeExpr2]] = !DISubrange(lowerBound: !DIExpression(), upperBound: !DIExpression()) ; CHECK-LLVM: !DISubrange(count: 1000, lowerBound: 1) ; CHECK-LLVM: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[#BaseType:]], elements: ![[#Subrage:]]) ; CHECK-LLVM: ![[#BaseType]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) ; CHECK-LLVM: ![[#Subrage]] = !{![[#Subrage:]]} ; CHECK-LLVM: ![[#Subrage]] = !DISubrange(count: -1 ; ModuleID = 'DebugInfoSubrangeUpperBound.bc' target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024" target triple = "spir64-unknown-unknown" %structtype = type { [72 x i1] } %"QNCA_a0$float" = type { float addrspace(4)*, i64, i64, i64, i64, i64, [1 x %structtype2] } %structtype2 = type { i64, i64, i64 } ; Function Attrs: noinline nounwind define spir_kernel void @__omp_offloading_811_198142f_random_fill_sp_l25(i32 addrspace(1)* noalias %0, %structtype* byval(%structtype) %"ascast$val", [1000 x i32] addrspace(1)* noalias %"ascastB$val") #0 !kernel_arg_addr_space !9 !kernel_arg_access_qual !10 !kernel_arg_type !11 !kernel_arg_type_qual !12 !kernel_arg_base_type !11 { newFuncRoot: %.ascast = bitcast %structtype* %"ascast$val" to %"QNCA_a0$float"* call void @llvm.dbg.value(metadata %"QNCA_a0$float"* %.ascast, metadata !13, metadata !DIExpression(DW_OP_deref)), !dbg !27 call void @llvm.dbg.value(metadata %"QNCA_a0$float"* %.ascast, metadata !28, metadata !DIExpression(DW_OP_deref)), !dbg !42 call void @llvm.dbg.value(metadata [1000 x i32] addrspace(1)* %"ascastB$val", metadata !47, metadata !DIExpression(DW_OP_deref)), !dbg !51 call void @llvm.dbg.value(metadata i32 addrspace(1)* %0, metadata !54, metadata !DIExpression(DW_OP_deref)), !dbg !59 ret void } ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { noinline nounwind } attributes #1 = { nofree nosync nounwind readnone speculatable willreturn } !llvm.module.flags = !{!0, !1} !llvm.dbg.cu = !{!2} !spirv.MemoryModel = !{!4} !opencl.enable.FP_CONTRACT = !{} !spirv.Source = !{!5} !opencl.spir.version = !{!6} !opencl.ocl.version = !{!6} !opencl.used.extensions = !{!7} !opencl.used.optional.core.features = !{!7} !spirv.Generator = !{!8} !0 = !{i32 7, !"Dwarf Version", i32 4} !1 = !{i32 2, !"Debug Info Version", i32 3} !2 = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: !3, producer: "Fortran", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug) !3 = !DIFile(filename: "f.f90", directory: "Fortran") !4 = !{i32 2, i32 2} !5 = !{i32 4, i32 200000} !6 = !{i32 2, i32 0} !7 = !{} !8 = !{i16 6, i16 14} !9 = !{i32 0} !10 = !{!"none"} !11 = !{!"structtype"} !12 = !{!""} !13 = !DILocalVariable(name: "a", scope: !14, file: !3, line: 15, type: !18) !14 = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split", scope: null, file: !3, line: 25, type: !15, scopeLine: 25, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !17) !15 = !DISubroutineType(types: !16) !16 = !{null} !17 = !{!13} !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64) !19 = !DICompositeType(tag: DW_TAG_array_type, baseType: !20, size: 32, elements: !21) !20 = !DIBasicType(name: "REAL*4", size: 32, encoding: DW_ATE_float) !21 = !{!22} !22 = !DISubrange(lowerBound: 1, upperBound: !23) !23 = !DILocalVariable(name: "A$1$upperbound", scope: !24, type: !26, flags: DIFlagArtificial) !24 = distinct !DISubprogram(name: "random_fill_sp", linkageName: "random_fill_sp", scope: null, file: !3, line: 15, type: !15, scopeLine: 15, spFlags: DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !25) !25 = !{!23} !26 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed) !27 = !DILocation(line: 15, column: 67, scope: !14) !28 = !DILocalVariable(name: "a", scope: !29, file: !3, line: 15, type: !33) !29 = distinct !DISubprogram(name: "random_fill_sp.DIR.OMP.TARGET.8.split.split.split.split", scope: null, file: !3, line: 25, type: !30, scopeLine: 25, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !32) !30 = !DISubroutineType(types: !31) !31 = !{null} !32 = !{!28} !33 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !34, size: 64) !34 = !DICompositeType(tag: DW_TAG_array_type, baseType: !35, size: 32, elements: !36) !35 = !DIBasicType(name: "REAL*4", size: 32, encoding: DW_ATE_float) !36 = !{!37} !37 = !DISubrange(lowerBound: !DIExpression(), upperBound: !DIExpression()) !38 = !DILocalVariable(name: "A$1$upperbound", scope: !39, type: !41, flags: DIFlagArtificial) !39 = distinct !DISubprogram(name: "random_fill_sp", linkageName: "random_fill_sp", scope: null, file: !3, line: 15, type: !30, scopeLine: 15, spFlags: DISPFlagDefinition, unit: !2, templateParams: !7, retainedNodes: !40) !40 = !{!38} !41 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed) !42 = !DILocation(line: 15, column: 67, scope: !29) !43 = !DIBasicType(name: "INTEGER*4", size: 32, encoding: DW_ATE_signed) !44 = !{} !45 = !DISubroutineType(types: !44) !46 = distinct !DISubprogram(name: "test_target_map_array_default_IP_test_array_map_no_map_type_.DIR.OMP.TARGET.340.split", scope: !3, file: !3, line: 32, type: !45, scopeLine: 32, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2) !47 = !DILocalVariable(name: "compute_array", scope: !46, file: !3, line: 27, type: !48) !48 = !DICompositeType(tag: DW_TAG_array_type, baseType: !43, elements: !49) !49 = !{!50} !50 = !DISubrange(count: 1000, lowerBound: 1) !51 = !DILocation(line: 27, column: 24, scope: !46) !52 = distinct !DISubprogram(name: "test", scope: !3, file: !3, line: 51, type: !53, scopeLine: 51, flags: DIFlagArtificial, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2) !53 = !DISubroutineType(types: !7) !54 = !DILocalVariable(name: "isHost", scope: !52, file: !3, line: 34, type: !55) !55 = !DICompositeType(tag: DW_TAG_array_type, baseType: !56, elements: !57) !56 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !57 = !{!58} !58 = !DISubrange(count: -1) !59 = !DILocation(line: 34, column: 33, scope: !52) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/RecursiveDebugInfo.ll000066400000000000000000000326411506646577400244250ustar00rootroot00000000000000; Test checks that the translator converts debug info correctly and ; doesn't crash if the module has recursive template parameters definition. ; This LLVM IR was generated using Intel SYCL Clang compiler (https://github.com/intel/llvm) ; recursive_debug_info.cpp: ; ; namespace s = cl::sycl; ; ; template ; struct iterator { ; int *Itr; ; iterator() : Itr(nullptr) {} ; iterator(int* itr) : Itr(itr) {} ; }; ; ; struct vector { ; int *Start; ; typedef iterator vec_it; ; vec_it begin() { ; return vec_it(Start); ; } ; }; ; ; class foo; ; ; int main() { ; s::queue q; ; auto e = q.submit([=](s::handler &cgh) { ; cgh.single_task([=]() { ; iterator IV; ; vector V; ; }); ; }); ; e.wait(); ; return 0; ; } ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s ; CHECK: [[IT_VEC:![0-9]+]] = !DICompositeType(tag: DW_TAG_structure_type, name: "iterator", {{.+}}, templateParams: [[TMPL_P:![0-9]+]] ; CHECK: [[TMPL_P]] = !{[[TMPL_P1:![0-9]+]]} ; CHECK: [[TMPL_P1]] = !DITemplateTypeParameter(name: "Container", type: [[CTNR_TY:![0-9]+]]) ; CHECK: [[CTNR_TY]] = !DICompositeType(tag: DW_TAG_structure_type, name: "vector", {{.+}}, elements: [[ELMS:![0-9]+]] ; CHECK: [[ELMS]] = !{!{{[0-9]+}}, [[EL2:![0-9]+]]} ; CHECK: [[EL2]] = !DISubprogram(name: "begin", {{.+}}, type: [[SPRG_TY:![0-9]+]] ; CHECK: [[SPRG_TY]] = !DISubroutineType(types: [[FNC_TYS:![0-9]+]]) ; CHECK: [[FNC_TYS]] = !{[[FNC_TY1:![0-9]+]], !{{[0-9]+}}} ; CHECK: [[FNC_TY1]] = !DIDerivedType(tag: DW_TAG_typedef, name: "vec_it", {{.+}}, baseType: [[IT_VEC]]) target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64-unknown-unknown" %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" = type { i8 } %struct._ZTS8iteratorI6vectorE.iterator = type { i32 addrspace(4)* } %struct._ZTS6vector.vector = type { i32 addrspace(4)* } $_ZTS3foo = comdat any $_ZN8iteratorI6vectorEC2Ev = comdat any define weak_odr dso_local spir_kernel void @_ZTS3foo() #0 comdat !dbg !12 !kernel_arg_addr_space !10 !kernel_arg_access_qual !10 !kernel_arg_type !10 !kernel_arg_base_type !10 !kernel_arg_type_qual !10 !kernel_arg_host_accessible !10 !kernel_arg_pipe_depth !10 !kernel_arg_pipe_io !10 !kernel_arg_buffer_location !10 { entry: %0 = alloca %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon", align 1 %1 = bitcast %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon"* %0 to i8* call void @llvm.lifetime.start.p0i8(i64 1, i8* %1) #5 call void @llvm.dbg.declare(metadata %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon"* %0, metadata !15, metadata !DIExpression()), !dbg !23 %2 = addrspacecast %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon"* %0 to %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)*, !dbg !24 call spir_func void @"_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlvE_clEv"(%"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)* %2), !dbg !24 %3 = bitcast %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon"* %0 to i8*, !dbg !23 call void @llvm.lifetime.end.p0i8(i64 1, i8* %3) #5, !dbg !23 ret void } ; Function Attrs: argmemonly nounwind willreturn declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 ; Function Attrs: nounwind readnone speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #2 ; Function Attrs: inlinehint define internal spir_func void @"_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlvE_clEv"(%"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)* %this) #3 align 2 !dbg !26 { entry: %this.addr = alloca %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)*, align 8 %IV = alloca %struct._ZTS8iteratorI6vectorE.iterator, align 8 %V = alloca %struct._ZTS6vector.vector, align 8 store %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)* %this, %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)** %this.addr, align 8, !tbaa !52 call void @llvm.dbg.declare(metadata %"class._ZTSZZ4mainENK3$_0clERN2cl4sycl7handlerEEUlvE_.anon" addrspace(4)** %this.addr, metadata !28, metadata !DIExpression()), !dbg !56 %0 = bitcast %struct._ZTS8iteratorI6vectorE.iterator* %IV to i8*, !dbg !57 call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #5, !dbg !57 call void @llvm.dbg.declare(metadata %struct._ZTS8iteratorI6vectorE.iterator* %IV, metadata !30, metadata !DIExpression()), !dbg !58 %1 = addrspacecast %struct._ZTS8iteratorI6vectorE.iterator* %IV to %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)*, !dbg !58 call spir_func void @_ZN8iteratorI6vectorEC2Ev(%struct._ZTS8iteratorI6vectorE.iterator addrspace(4)* %1), !dbg !58 %2 = bitcast %struct._ZTS6vector.vector* %V to i8*, !dbg !59 call void @llvm.lifetime.start.p0i8(i64 8, i8* %2) #5, !dbg !59 call void @llvm.dbg.declare(metadata %struct._ZTS6vector.vector* %V, metadata !51, metadata !DIExpression()), !dbg !60 %3 = bitcast %struct._ZTS6vector.vector* %V to i8*, !dbg !61 call void @llvm.lifetime.end.p0i8(i64 8, i8* %3) #5, !dbg !61 %4 = bitcast %struct._ZTS8iteratorI6vectorE.iterator* %IV to i8*, !dbg !61 call void @llvm.lifetime.end.p0i8(i64 8, i8* %4) #5, !dbg !61 ret void, !dbg !61 } ; Function Attrs: argmemonly nounwind willreturn declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 ; Function Attrs: nounwind define linkonce_odr dso_local spir_func void @_ZN8iteratorI6vectorEC2Ev(%struct._ZTS8iteratorI6vectorE.iterator addrspace(4)* %this) unnamed_addr #4 comdat align 2 !dbg !62 { entry: %this.addr = alloca %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)*, align 8 store %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)* %this, %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)** %this.addr, align 8, !tbaa !52 call void @llvm.dbg.declare(metadata %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)** %this.addr, metadata !64, metadata !DIExpression()), !dbg !66 %this1 = load %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)*, %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)** %this.addr, align 8 %Itr = getelementptr inbounds %struct._ZTS8iteratorI6vectorE.iterator, %struct._ZTS8iteratorI6vectorE.iterator addrspace(4)* %this1, i32 0, i32 0, !dbg !67 store i32 addrspace(4)* null, i32 addrspace(4)* addrspace(4)* %Itr, align 8, !dbg !67, !tbaa !68 ret void, !dbg !70 } attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "sycl-module-id"="recursive_debug_info.cpp" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { argmemonly nounwind willreturn } attributes #2 = { nounwind readnone speculatable willreturn } attributes #3 = { inlinehint "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #4 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #5 = { nounwind } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!5, !6, !7} !opencl.spir.version = !{!8} !spirv.Source = !{!9} !opencl.used.extensions = !{!10} !opencl.used.optional.core.features = !{!10} !opencl.compiler.options = !{!10} !llvm.ident = !{!11} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, nameTableKind: None) !1 = !DIFile(filename: "recursive_debug_info.cpp", directory: "/localdisk/test") !2 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !3 = !{null} !4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !2, size: 64) !5 = !{i32 7, !"Dwarf Version", i32 4} !6 = !{i32 2, !"Debug Info Version", i32 3} !7 = !{i32 1, !"wchar_size", i32 4} !8 = !{i32 1, i32 2} !9 = !{i32 4, i32 100000} !10 = !{} !11 = !{!"clang version 10.0.0"} !12 = distinct !DISubprogram(name: "_ZTS3foo", scope: !1, file: !1, line: 28, type: !13, flags: DIFlagArtificial | DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14) !13 = !DISubroutineType(cc: DW_CC_LLVM_OpenCLKernel, types: !3) !14 = !{!15} !15 = !DILocalVariable(scope: !12, file: !1, type: !16) !16 = distinct !DICompositeType(tag: DW_TAG_class_type, file: !1, line: 28, size: 8, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !17) !17 = !{!18} !18 = !DISubprogram(name: "operator()", scope: !16, file: !1, line: 28, type: !19, scopeLine: 28, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagOptimized) !19 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !20) !20 = !{null, !21} !21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !22 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !16) !23 = !DILocation(line: 0, scope: !12) !24 = !DILocation(line: 0, scope: !25) !25 = distinct !DILexicalBlock(scope: !12, file: !1) !26 = distinct !DISubprogram(name: "operator()", linkageName: "_ZZZ4mainENK3$_0clERN2cl4sycl7handlerEENKUlvE_clEv", scope: !16, file: !1, line: 28, type: !19, scopeLine: 28, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !18, retainedNodes: !27) !27 = !{!28, !30, !51} !28 = !DILocalVariable(name: "this", arg: 1, scope: !26, type: !29, flags: DIFlagArtificial | DIFlagObjectPointer) !29 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !22, size: 64) !30 = !DILocalVariable(name: "IV", scope: !26, file: !1, line: 29, type: !31) !31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "iterator", file: !1, line: 6, size: 64, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !32, templateParams: !41, identifier: "_ZTS8iteratorI6vectorE") !32 = !{!33, !34, !38} !33 = !DIDerivedType(tag: DW_TAG_member, name: "Itr", scope: !31, file: !1, line: 7, baseType: !4, size: 64) !34 = !DISubprogram(name: "iterator", scope: !31, file: !1, line: 9, type: !35, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized) !35 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !36) !36 = !{null, !37} !37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !31, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !38 = !DISubprogram(name: "iterator", scope: !31, file: !1, line: 10, type: !39, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized) !39 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !40) !40 = !{null, !37, !4} !41 = !{!42} !42 = !DITemplateTypeParameter(name: "Container", type: !43) !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "vector", file: !1, line: 13, size: 64, flags: DIFlagTypePassByValue, elements: !44, identifier: "_ZTS6vector") !44 = !{!45, !46} !45 = !DIDerivedType(tag: DW_TAG_member, name: "Start", scope: !43, file: !1, line: 14, baseType: !4, size: 64) !46 = !DISubprogram(name: "begin", linkageName: "_ZN6vector5beginEv", scope: !43, file: !1, line: 18, type: !47, scopeLine: 18, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized) !47 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !48) !48 = !{!49, !50} !49 = !DIDerivedType(tag: DW_TAG_typedef, name: "vec_it", scope: !43, file: !1, line: 16, baseType: !31) !50 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !43, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer) !51 = !DILocalVariable(name: "V", scope: !26, file: !1, line: 30, type: !43) !52 = !{!53, !53, i64 0} !53 = !{!"any pointer", !54, i64 0} !54 = !{!"omnipotent char", !55, i64 0} !55 = !{!"Simple C++ TBAA"} !56 = !DILocation(line: 0, scope: !26) !57 = !DILocation(line: 29, column: 7, scope: !26) !58 = !DILocation(line: 29, column: 24, scope: !26) !59 = !DILocation(line: 30, column: 7, scope: !26) !60 = !DILocation(line: 30, column: 14, scope: !26) !61 = !DILocation(line: 31, column: 5, scope: !26) !62 = distinct !DISubprogram(name: "iterator", linkageName: "_ZN8iteratorI6vectorEC2Ev", scope: !31, file: !1, line: 9, type: !35, scopeLine: 9, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !34, retainedNodes: !63) !63 = !{!64} !64 = !DILocalVariable(name: "this", arg: 1, scope: !62, type: !65, flags: DIFlagArtificial | DIFlagObjectPointer) !65 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !31, size: 64) !66 = !DILocation(line: 0, scope: !62) !67 = !DILocation(line: 9, column: 16, scope: !62) !68 = !{!69, !53, i64 0} !69 = !{!"_ZTS8iteratorI6vectorE", !53, i64 0} !70 = !DILocation(line: 9, column: 30, scope: !62) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/RelativeSrcPath.ll000066400000000000000000000062031506646577400237260ustar00rootroot00000000000000; Source: ; __kernel void foo(__global int *a, __global int *b) { ; a[0] += b[0]; ; } ; Command: ; clang -cc1 -triple spir -O0 -debug-info-kind=line-tables-only -emit-llvm -o RelativeSrcPath.ll RelativeSrcPath.cl ; Directory: /tmp ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s ; ModuleID = 'RelativeSrcPath.cl' source_filename = "RelativeSrcPath.cl" target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir" ; Function Attrs: convergent noinline nounwind optnone define spir_kernel void @foo(i32 addrspace(1)* %a, i32 addrspace(1)* %b) #0 !dbg !8 !kernel_arg_addr_space !11 !kernel_arg_access_qual !12 !kernel_arg_type !13 !kernel_arg_base_type !13 !kernel_arg_type_qual !14 { entry: %a.addr = alloca i32 addrspace(1)*, align 4 %b.addr = alloca i32 addrspace(1)*, align 4 store i32 addrspace(1)* %a, i32 addrspace(1)** %a.addr, align 4 store i32 addrspace(1)* %b, i32 addrspace(1)** %b.addr, align 4 %0 = load i32 addrspace(1)*, i32 addrspace(1)** %b.addr, align 4, !dbg !15 %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %0, i32 0, !dbg !15 %1 = load i32, i32 addrspace(1)* %arrayidx, align 4, !dbg !15 %2 = load i32 addrspace(1)*, i32 addrspace(1)** %a.addr, align 4, !dbg !15 %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %2, i32 0, !dbg !15 %3 = load i32, i32 addrspace(1)* %arrayidx1, align 4, !dbg !15 %add = add nsw i32 %3, %1, !dbg !15 store i32 %add, i32 addrspace(1)* %arrayidx1, align 4, !dbg !15 ret void, !dbg !16 } attributes #0 = { convergent noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !opencl.ocl.version = !{!5} !opencl.spir.version = !{!6} !llvm.ident = !{!7} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (cfe/trunk)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 1, i32 0} !6 = !{i32 1, i32 2} !7 = !{!"clang version 8.0.0 (cfe/trunk)"} !8 = distinct !DISubprogram(name: "foo", scope: !9, file: !9, line: 1, type: !10, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) ; CHECK: String [[ID:[0-9]+]] "/tmp{{[/\\]}}RelativeSrcPath.cl" ; CHECK: Line [[ID]] !9 = !DIFile(filename: "RelativeSrcPath.cl", directory: "/tmp") !10 = !DISubroutineType(types: !2) !11 = !{i32 1, i32 1} !12 = !{!"none", !"none"} !13 = !{!"int*", !"int*"} !14 = !{!"", !""} !15 = !DILocation(line: 2, scope: !8) !16 = !DILocation(line: 3, scope: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/SourceLanguageLLVMToSPIRV.ll000066400000000000000000000043241506646577400254160ustar00rootroot00000000000000; Test checks that DW_LANG_C99 and DW_LANG_OpenCL are mapped to OpenCL C ; in the SourceLanguage enum, and also that DW_LANG_C_plus_plus and ; DW_LANG_C_plus_plus_14 are mapped to C++ for OpenCL. ; LLVM does not yet define DW_LANG_C_plus_plus_17. A test case that ; checks DW_LANG_C_plus_plus_17 is mapped to C++ for OpenCL should be ; added once this is defined. ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C99/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-OPENCLC ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_OpenCL/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-OPENCLC ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C_plus_plus/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-CPP4OPENCL ; RUN: sed -e 's/INPUT_LANGUAGE/DW_LANG_C_plus_plus_14/' %s | llvm-as - -o %t.bc ; RUN: llvm-spirv -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-CPP4OPENCL target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir64-unknown-unknown" define dso_local spir_kernel void @func() local_unnamed_addr !dbg !7 !kernel_arg_addr_space !2 !kernel_arg_access_qual !2 !kernel_arg_type !2 !kernel_arg_base_type !2 !kernel_arg_type_qual !2 { entry: ret void } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !0 = distinct !DICompileUnit(language: INPUT_LANGUAGE, file: !1) !1 = !DIFile(filename: "test.cl", directory: "/tmp", checksumkind: CSK_MD5, checksum: "18aa9ce738eaafc7b7b7181c19092815") !2 = !{} !3 = !{i32 7, !"Dwarf Version", i32 5} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{i32 2, i32 0} !7 = distinct !DISubprogram(name: "func", scope: !8, file: !8, line: 1, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2) !8 = !DIFile(filename: "test.cl", directory: "/tmp", checksumkind: CSK_MD5, checksum: "18aa9ce738eaafc7b7b7181c19092815") ; CHECK-OPENCLC: DebugCompilationUnit {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 3 ; CHECK-CPP4OPENCL: DebugCompilationUnit {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} 6 SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/SourceLanguageSPIRVToLLVM.spvasm000066400000000000000000000061301506646577400263150ustar00rootroot00000000000000; Test checks that: ; - OpenCL_CPP is mapped to DW_LANG_C_plus_plus_14 ; - CPP_for_OpenCL is mapped to DW_LANG_C_plus_plus_17 ; - OpenCL_C, GLSL, ESSL, HLSL, and Unknown are mapped to DW_LANG_OpenCL ; LLVM does not yet define DW_LANG_C_plus_plus_17. When it is defined, the ; test case for C++ for OpenCL should be updated to check that CPP_for_OpenCL ; is mapped to DW_LANG_C_plus_plus_17, not DW_LANG_C_plus_plus_14. ; REQUIRES: spirv-as ; RUN: sed -e 's/SOURCE_LANGUAGE/OpenCL_CPP/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-CPP14 ; COM: TODO: Enable this test when the version of SPIRV-Tools used by buildbot is updated to ; COM: recognise CPP_for_OpenCL as a valid source language. ; COM: sed -e 's/SOURCE_LANGUAGE/CPP_for_OpenCL/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; COM: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-CPP14 ; RUN: sed -e 's/SOURCE_LANGUAGE/OpenCL_C/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-OPENCL ; RUN: sed -e 's/SOURCE_LANGUAGE/GLSL/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-OPENCL ; RUN: sed -e 's/SOURCE_LANGUAGE/HLSL/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-OPENCL ; RUN: sed -e 's/SOURCE_LANGUAGE/ESSL/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-OPENCL ; RUN: sed -e 's/SOURCE_LANGUAGE/Unknown/' %s | spirv-as --target-env spv1.3 - -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-OPENCL ; SPIR-V ; Version: 1.1 ; Generator: Khronos LLVM/SPIR-V Translator; 14 ; Bound: 16 ; Schema: 0 OpCapability Addresses OpCapability Kernel %1 = OpExtInstImport "OpenCL.std" %2 = OpExtInstImport "OpenCL.DebugInfo.100" OpMemoryModel Physical64 OpenCL OpEntryPoint Kernel %5 "func" %7 = OpString "kernel_arg_type.func." %8 = OpString "/tmp/test.cl" %9 = OpString "//__CSK_MD5:18aa9ce738eaafc7b7b7181c19092815" %12 = OpString "func" %14 = OpString "" OpSource Unknown 0 OpName %entry "entry" OpModuleProcessed "Debug info producer: " %void = OpTypeVoid %4 = OpTypeFunction %void %10 = OpExtInst %void %2 DebugSource %8 %9 %11 = OpExtInst %void %2 DebugCompilationUnit 65536 5 %10 SOURCE_LANGUAGE %13 = OpExtInst %void %2 DebugInfoNone %15 = OpExtInst %void %2 DebugFunction %12 %13 %10 1 0 %11 %14 FlagIsDefinition|FlagPrototyped|FlagIsOptimized 2 %5 %13 %5 = OpFunction %void None %4 %entry = OpLabel OpReturn OpFunctionEnd ; CHECK-OPENCL: !DICompileUnit(language: DW_LANG_OpenCL, ; CHECK-CPP14: !DICompileUnit(language: DW_LANG_C_plus_plus_14, SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/TransTypeCompositeCaseClass_.ll000066400000000000000000000072471506646577400264340ustar00rootroot00000000000000; Source code: ; class A {}; ; struct B {}; ; int main() { ; A a; ; B b; ; return 0; ; } ; RUN: llvm-as %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; RUN: llvm-spirv --spirv-debug-info-version=nonsemantic-shader-100 %t.bc -o %t.spv ; RUN: llvm-spirv -r -emit-opaque-pointers %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM ; ModuleID = 'main.cpp' source_filename = "main.cpp" target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64" %class.A = type { i8 } %struct.B = type { i8 } ; Function Attrs: noinline norecurse nounwind optnone mustprogress define dso_local i32 @main() #0 !dbg !6 { entry: %retval = alloca i32, align 4 %a = alloca %class.A, align 1 %b = alloca %struct.B, align 1 store i32 0, i32* %retval, align 4 call void @llvm.dbg.declare(metadata %class.A* %a, metadata !11, metadata !DIExpression()), !dbg !13 call void @llvm.dbg.declare(metadata %struct.B* %b, metadata !14, metadata !DIExpression()), !dbg !16 ret i32 0, !dbg !17 } ; Function Attrs: nofree nosync nounwind readnone speculatable willreturn declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { noinline norecurse nounwind optnone mustprogress "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nofree nosync nounwind readnone speculatable willreturn } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !opencl.used.extensions = !{!2} !opencl.used.optional.core.features = !{!2} !opencl.compiler.options = !{!2} !llvm.ident = !{!5} ; CHECK-LLVM: !DICompositeType ; CHECK-LLVM-SAME: tag: DW_TAG_class_type ; CHECK-LLVM-SAME: name: "A" ; CHECK-LLVM: !DICompositeType ; CHECK-LLVM-SAME: tag: DW_TAG_structure_type ; CHECK-LLVM-SAME: name: "B" !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "/export/users") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{!"clang version 13.0.0 (https://github.com/llvm/llvm-project.git 16a50c9e642fd085e5ceb68c403b71b5b2e0607c)"} !6 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 4, type: !8, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !7 = !DIFile(filename: "main.cpp", directory: "/export/users") !8 = !DISubroutineType(types: !9) !9 = !{!10} !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !11 = !DILocalVariable(name: "a", scope: !6, file: !7, line: 5, type: !12) !12 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !7, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS1A") !13 = !DILocation(line: 5, column: 7, scope: !6) !14 = !DILocalVariable(name: "b", scope: !6, file: !7, line: 6, type: !15) !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", file: !7, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS1B") !16 = !DILocation(line: 6, column: 7, scope: !6) !17 = !DILocation(line: 7, column: 5, scope: !6) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/UnknownBaseType.ll000066400000000000000000000052141506646577400237630ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll -filetype=obj | llvm-dwarfdump -debug-info - | FileCheck %s ; Check that translator doesn't crash when it encounter basic type encoding ; (e.g. DW_ATE_complex_float) which is missing in the spec. ; CHECK: DW_TAG_unspecified_type ; CHECK-NEXT: DW_AT_name ("complex") target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" target triple = "spir-unknown-unknown" ; Function Attrs: convergent nounwind define spir_func void @foo({ float, float }* byval({ float, float }) align 4 %f) #0 !dbg !7 { entry: call void @llvm.dbg.declare(metadata { float, float }* %f, metadata !13, metadata !DIExpression()), !dbg !14 ret void, !dbg !14 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { convergent nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4} !opencl.ocl.version = !{!5} !opencl.spir.version = !{!5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (https://llvm.org/git/clang 1b09e8845172eccc47c896f546fa30805da53d51) (https://llvm.org/git/llvm 384f64397f6ad95a361b72d62c07d7bac9f24163)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = !{i32 1, !"wchar_size", i32 4} !5 = !{i32 2, i32 0} !6 = !{!"clang version 9.0.0"} !7 = distinct !DISubprogram(name: "foo", scope: !8, file: !8, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12) !8 = !DIFile(filename: "tmp.cl", directory: "/tmp") !9 = !DISubroutineType(cc: DW_CC_LLVM_SpirFunction, types: !10) !10 = !{null, !11} !11 = !DIBasicType(name: "complex", size: 64, encoding: DW_ATE_complex_float) !12 = !{!13} !13 = !DILocalVariable(name: "f", arg: 1, scope: !7, file: !8, line: 1, type: !11) !14 = !DILocation(line: 1, scope: !7) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/000077500000000000000000000000001506646577400207215ustar00rootroot00000000000000SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/2010-04-13-PubType.ll000066400000000000000000000065421506646577400236730ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -O0 -asm-verbose -mtriple=x86_64-macosx -debugger-tune=gdb < %t.ll | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK-NOT: .asciz "X" ## External Name ; CHECK: .asciz "Y" ## External Name ; Test to check type with no definition is listed in pubtypes section. %struct.X = type opaque %struct.Y = type { i32 } define i32 @foo(%struct.X* %x, %struct.Y* %y) nounwind ssp !dbg !1 { entry: %x_addr = alloca %struct.X* ; <%struct.X**> [#uses=1] %y_addr = alloca %struct.Y* ; <%struct.Y**> [#uses=1] %retval = alloca i32 ; [#uses=2] %0 = alloca i32 ; [#uses=2] %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.declare(metadata %struct.X** %x_addr, metadata !0, metadata !DIExpression()), !dbg !13 store %struct.X* %x, %struct.X** %x_addr call void @llvm.dbg.declare(metadata %struct.Y** %y_addr, metadata !14, metadata !DIExpression()), !dbg !13 store %struct.Y* %y, %struct.Y** %y_addr store i32 0, i32* %0, align 4, !dbg !13 %1 = load i32, i32* %0, align 4, !dbg !13 ; [#uses=1] store i32 %1, i32* %retval, align 4, !dbg !13 br label %return, !dbg !13 return: ; preds = %entry %retval1 = load i32, i32* %retval, !dbg !13 ; [#uses=1] ret i32 %retval1, !dbg !15 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!20} !0 = !DILocalVariable(name: "x", line: 7, arg: 1, scope: !1, file: !2, type: !7) !1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scopeLine: 7, file: !18, scope: !2, type: !4) !2 = !DIFile(filename: "a.c", directory: "/tmp/") !3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: FullDebug, file: !18, enums: !19, retainedTypes: !19, imports: null) !4 = !DISubroutineType(types: !5) !5 = !{!6, !7, !9} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !8) !8 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", line: 3, flags: DIFlagFwdDecl, file: !18, scope: !2) !9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !18, scope: !2, baseType: !10) !10 = !DICompositeType(tag: DW_TAG_structure_type, name: "Y", line: 4, size: 32, align: 32, file: !18, scope: !2, elements: !11) !11 = !{!12} !12 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 5, size: 32, align: 32, file: !18, scope: !10, baseType: !6) !13 = !DILocation(line: 7, scope: !1) !14 = !DILocalVariable(name: "y", line: 7, arg: 2, scope: !1, file: !2, type: !9) !15 = !DILocation(line: 7, scope: !16) !16 = distinct !DILexicalBlock(line: 7, column: 0, file: !18, scope: !1) !18 = !DIFile(filename: "a.c", directory: "/tmp/") !19 = !{} !20 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll000066400000000000000000000054541506646577400255340ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-pc-linux-gnu %t.ll -o %t -filetype=obj ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ModuleID = 'test.c' source_filename = "test/DebugInfo/X86/2011-09-26-GlobalVarContext.ll" @GLB = common global i32 0, align 4, !dbg !0 ; Function Attrs: nounwind define i32 @f() #0 !dbg !8 { %LOC = alloca i32, align 4 call void @llvm.dbg.declare(metadata i32* %LOC, metadata !11, metadata !13), !dbg !14 %1 = load i32, i32* @GLB, align 4, !dbg !15 store i32 %1, i32* %LOC, align 4, !dbg !15 %2 = load i32, i32* @GLB, align 4, !dbg !16 ret i32 %2, !dbg !16 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!4} !llvm.module.flags = !{!7} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "GLB", scope: null, file: !2, line: 1, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "test.c", directory: "/work/llvm/vanilla/test/DebugInfo") !3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !4 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, producer: "clang version 3.0 (trunk)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6, imports: !5) !5 = !{} !6 = !{!0} !7 = !{i32 1, !"Debug Info Version", i32 3} !8 = distinct !DISubprogram(name: "f", scope: !2, file: !2, line: 3, type: !9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !4) !9 = !DISubroutineType(types: !10) !10 = !{!3} !11 = !DILocalVariable(name: "LOC", scope: !12, file: !2, line: 4, type: !3) !12 = distinct !DILexicalBlock(scope: !8, file: !2, line: 3, column: 9) !13 = !DIExpression() !14 = !DILocation(line: 4, column: 9, scope: !12) !15 = !DILocation(line: 4, column: 23, scope: !12) !16 = !DILocation(line: 5, column: 5, scope: !12) ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name [DW_FORM_strp] ( .debug_str[0x{{[0-9a-f]*}}] = "GLB") ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_decl_file [DW_FORM_data1] ("/work/llvm/vanilla/test/DebugInfo{{[/\\]}}test.c") ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_decl_line [DW_FORM_data1] (1) ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name [DW_FORM_strp] ( .debug_str[0x{{[0-9a-f]*}}] = "LOC") ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_decl_file [DW_FORM_data1] ("/work/llvm/vanilla/test/DebugInfo{{[/\\]}}test.c") ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_decl_line [DW_FORM_data1] (4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/2011-12-16-BadStructRef.ll000066400000000000000000000244301506646577400246320ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-macosx10.7 %t.ll -o %t -filetype=obj ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: b_ref ; CHECK-NOT: AT_bit_size %struct.bar = type { %struct.baz, %struct.baz* } %struct.baz = type { i32 } define i32 @main(i32 %argc, i8** %argv) uwtable ssp !dbg !29 { entry: %retval = alloca i32, align 4 %argc.addr = alloca i32, align 4 %argv.addr = alloca i8**, align 8 %myBar = alloca %struct.bar, align 8 store i32 0, i32* %retval store i32 %argc, i32* %argc.addr, align 4 call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !49, metadata !DIExpression()), !dbg !50 store i8** %argv, i8*** %argv.addr, align 8 call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !51, metadata !DIExpression()), !dbg !52 call void @llvm.dbg.declare(metadata %struct.bar* %myBar, metadata !53, metadata !DIExpression()), !dbg !55 call void @_ZN3barC1Ei(%struct.bar* %myBar, i32 1), !dbg !56 ret i32 0, !dbg !57 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone define linkonce_odr void @_ZN3barC1Ei(%struct.bar* %this, i32 %x) unnamed_addr uwtable ssp align 2 !dbg !37 { entry: %this.addr = alloca %struct.bar*, align 8 %x.addr = alloca i32, align 4 store %struct.bar* %this, %struct.bar** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !58, metadata !DIExpression()), !dbg !59 store i32 %x, i32* %x.addr, align 4 call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !60, metadata !DIExpression()), !dbg !61 %this1 = load %struct.bar*, %struct.bar** %this.addr %0 = load i32, i32* %x.addr, align 4, !dbg !62 call void @_ZN3barC2Ei(%struct.bar* %this1, i32 %0), !dbg !62 ret void, !dbg !62 } define linkonce_odr void @_ZN3barC2Ei(%struct.bar* %this, i32 %x) unnamed_addr uwtable ssp align 2 !dbg !40 { entry: %this.addr = alloca %struct.bar*, align 8 %x.addr = alloca i32, align 4 store %struct.bar* %this, %struct.bar** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.bar** %this.addr, metadata !63, metadata !DIExpression()), !dbg !64 store i32 %x, i32* %x.addr, align 4 call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !65, metadata !DIExpression()), !dbg !66 %this1 = load %struct.bar*, %struct.bar** %this.addr %b = getelementptr inbounds %struct.bar, %struct.bar* %this1, i32 0, i32 0, !dbg !67 %0 = load i32, i32* %x.addr, align 4, !dbg !67 call void @_ZN3bazC1Ei(%struct.baz* %b, i32 %0), !dbg !67 %1 = getelementptr inbounds %struct.bar, %struct.bar* %this1, i32 0, i32 1, !dbg !67 %b2 = getelementptr inbounds %struct.bar, %struct.bar* %this1, i32 0, i32 0, !dbg !67 store %struct.baz* %b2, %struct.baz** %1, align 8, !dbg !67 ret void, !dbg !68 } define linkonce_odr void @_ZN3bazC1Ei(%struct.baz* %this, i32 %a) unnamed_addr uwtable ssp align 2 !dbg !43 { entry: %this.addr = alloca %struct.baz*, align 8 %a.addr = alloca i32, align 4 store %struct.baz* %this, %struct.baz** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !70, metadata !DIExpression()), !dbg !71 store i32 %a, i32* %a.addr, align 4 call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !72, metadata !DIExpression()), !dbg !73 %this1 = load %struct.baz*, %struct.baz** %this.addr %0 = load i32, i32* %a.addr, align 4, !dbg !74 call void @_ZN3bazC2Ei(%struct.baz* %this1, i32 %0), !dbg !74 ret void, !dbg !74 } define linkonce_odr void @_ZN3bazC2Ei(%struct.baz* %this, i32 %a) unnamed_addr nounwind uwtable ssp align 2 !dbg !46 { entry: %this.addr = alloca %struct.baz*, align 8 %a.addr = alloca i32, align 4 store %struct.baz* %this, %struct.baz** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.baz** %this.addr, metadata !75, metadata !DIExpression()), !dbg !76 store i32 %a, i32* %a.addr, align 4 call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !77, metadata !DIExpression()), !dbg !78 %this1 = load %struct.baz*, %struct.baz** %this.addr %h = getelementptr inbounds %struct.baz, %struct.baz* %this1, i32 0, i32 0, !dbg !79 %0 = load i32, i32* %a.addr, align 4, !dbg !79 store i32 %0, i32* %h, align 4, !dbg !79 ret void, !dbg !80 } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!83} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 146596)", isOptimized: false, emissionKind: FullDebug, file: !82, enums: !1, retainedTypes: !3, globals: !1, imports: !1) !1 = !{} !3 = !{!5, !9} !5 = !DICompositeType(tag: DW_TAG_class_type, name: "bar", line: 9, size: 128, align: 64, file: !82, elements: !7) !6 = !DIFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref") !7 = !{!8, !19, !21} !8 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 11, size: 32, align: 32, file: !82, scope: !5, baseType: !9) !9 = !DICompositeType(tag: DW_TAG_class_type, name: "baz", line: 3, size: 32, align: 32, file: !82, elements: !10) !10 = !{!11, !13} !11 = !DIDerivedType(tag: DW_TAG_member, name: "h", line: 5, size: 32, align: 32, file: !82, scope: !9, baseType: !12) !12 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !13 = !DISubprogram(name: "baz", line: 6, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !9, type: !14) !14 = !DISubroutineType(types: !15) !15 = !{null, !16, !12} !16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !9) !19 = !DIDerivedType(tag: DW_TAG_member, name: "b_ref", line: 12, size: 64, align: 64, offset: 64, file: !82, scope: !5, baseType: !20) !20 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !9) !21 = !DISubprogram(name: "bar", line: 13, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !82, scope: !5, type: !22) !22 = !DISubroutineType(types: !23) !23 = !{null, !24, !12} !24 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, baseType: !5) !29 = distinct !DISubprogram(name: "main", line: 17, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !82, scope: !6, type: !30) !30 = !DISubroutineType(types: !31) !31 = !{!12, !12, !32} !32 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !33) !33 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !34) !34 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !35 = !{!36} !36 = !{} ; previously: invalid DW_TAG_base_type !37 = distinct !DISubprogram(name: "bar", linkageName: "_ZN3barC1Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !82, scope: null, type: !22, declaration: !21) !38 = !{!39} !39 = !{} ; previously: invalid DW_TAG_base_type !40 = distinct !DISubprogram(name: "bar", linkageName: "_ZN3barC2Ei", line: 13, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !82, scope: null, type: !22, declaration: !21) !41 = !{!42} !42 = !{} ; previously: invalid DW_TAG_base_type !43 = distinct !DISubprogram(name: "baz", linkageName: "_ZN3bazC1Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !82, scope: null, type: !14, declaration: !13) !44 = !{!45} !45 = !{} ; previously: invalid DW_TAG_base_type !46 = distinct !DISubprogram(name: "baz", linkageName: "_ZN3bazC2Ei", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !82, scope: null, type: !14, declaration: !13) !49 = !DILocalVariable(name: "argc", line: 16, arg: 1, scope: !29, file: !6, type: !12) !50 = !DILocation(line: 16, column: 14, scope: !29) !51 = !DILocalVariable(name: "argv", line: 16, arg: 2, scope: !29, file: !6, type: !32) !52 = !DILocation(line: 16, column: 27, scope: !29) !53 = !DILocalVariable(name: "myBar", line: 18, scope: !54, file: !6, type: !5) !54 = distinct !DILexicalBlock(line: 17, column: 1, file: !82, scope: !29) !55 = !DILocation(line: 18, column: 9, scope: !54) !56 = !DILocation(line: 18, column: 17, scope: !54) !57 = !DILocation(line: 19, column: 5, scope: !54) !58 = !DILocalVariable(name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !37, file: !6, type: !24) !59 = !DILocation(line: 13, column: 5, scope: !37) !60 = !DILocalVariable(name: "x", line: 13, arg: 2, scope: !37, file: !6, type: !12) !61 = !DILocation(line: 13, column: 13, scope: !37) !62 = !DILocation(line: 13, column: 34, scope: !37) !63 = !DILocalVariable(name: "this", line: 13, arg: 1, flags: DIFlagArtificial, scope: !40, file: !6, type: !24) !64 = !DILocation(line: 13, column: 5, scope: !40) !65 = !DILocalVariable(name: "x", line: 13, arg: 2, scope: !40, file: !6, type: !12) !66 = !DILocation(line: 13, column: 13, scope: !40) !67 = !DILocation(line: 13, column: 33, scope: !40) !68 = !DILocation(line: 13, column: 34, scope: !69) !69 = distinct !DILexicalBlock(line: 13, column: 33, file: !82, scope: !40) !70 = !DILocalVariable(name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !43, file: !6, type: !16) !71 = !DILocation(line: 6, column: 5, scope: !43) !72 = !DILocalVariable(name: "a", line: 6, arg: 2, scope: !43, file: !6, type: !12) !73 = !DILocation(line: 6, column: 13, scope: !43) !74 = !DILocation(line: 6, column: 24, scope: !43) !75 = !DILocalVariable(name: "this", line: 6, arg: 1, flags: DIFlagArtificial, scope: !46, file: !6, type: !16) !76 = !DILocation(line: 6, column: 5, scope: !46) !77 = !DILocalVariable(name: "a", line: 6, arg: 2, scope: !46, file: !6, type: !12) !78 = !DILocation(line: 6, column: 13, scope: !46) !79 = !DILocation(line: 6, column: 23, scope: !46) !80 = !DILocation(line: 6, column: 24, scope: !81) !81 = distinct !DILexicalBlock(line: 6, column: 23, file: !82, scope: !46) !82 = !DIFile(filename: "main.cpp", directory: "/Users/echristo/tmp/bad-struct-ref") !83 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DIModule.ll000066400000000000000000000047021506646577400227170ustar00rootroot00000000000000; ModuleID = '/Volumes/Data/apple-internal/llvm/tools/clang/test/Modules/debug-info-moduleimport.m' ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_debug_module %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-macosx %t.ll -accel-tables=Dwarf -o %t -filetype=obj ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s ; RUN: llvm-dwarfdump -verify %t ; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_debug_module %t.bc -spirv-text -o - | FileCheck %s --check-prefix CHECK-SPIRV ; CHECK: DW_TAG_compile_unit ; CHECK-NOT: DW_TAG ; CHECK: DW_TAG_module ; CHECK-NEXT: DW_AT_name {{.*}}"DebugModule" ; CHECK-NEXT: DW_AT_LLVM_config_macros {{.*}}"-DMODULES=0" ; CHECK-NEXT: DW_AT_LLVM_include_path {{.*}}"/llvm/tools/clang/test/Modules/Inputs" ; CHECK-NEXT: DW_AT_LLVM_apinotes {{.*}}"m.apinotes" ; CHECK-SPIRV: Capability DebugInfoModuleINTEL ; CHECK-SPIRV: Extension "SPV_INTEL_debug_module" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; CHECK-SPIRV: String [[FileName:[0-9]+]] "/llvm/tools/clang/test/Modules/" ; CHECK-SPIRV: String [[EmptyStr:[0-9]+]] "" ; CHECK-SPIRV: String [[Name:[0-9]+]] "DebugModule" ; CHECK-SPIRV: String [[Defines:[0-9]+]] "-DMODULES=0" ; CHECK-SPIRV: String [[IncludePath:[0-9]+]] "/llvm/tools/clang/test/Modules/Inputs" ; CHECK-SPIRV: String [[ApiNotes:[0-9]+]] "m.apinotes" ; CHECK-SPIRV: ExtInst {{[0-9]+}} [[Module:[0-9]+]] {{[0-9]+}} DebugModuleINTEL [[Name]] {{[0-9]+}} 0 {{[0-9]+}} [[Defines]] [[IncludePath]] [[ApiNotes]] 0 ; CHECK-SPIRV: ExtInst {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} DebugImportedEntity {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} {{[0-9]+}} [[Module]] !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!6, !7} !llvm.ident = !{!8} !0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "LLVM version 3.7.0", isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !3, sysroot: "/") !1 = !DIFile(filename: "/llvm/tools/clang/test/Modules/", directory: "/") !2 = !{} !3 = !{!4} !4 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !5, file: !1, line: 5) !5 = !DIModule(scope: null, name: "DebugModule", configMacros: "-DMODULES=0", includePath: "/llvm/tools/clang/test/Modules/Inputs", apinotes: "m.apinotes") !6 = !{i32 2, !"Dwarf Version", i32 4} !7 = !{i32 2, !"Debug Info Version", i32 3} !8 = !{!"LLVM version 3.7.0"} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DIModuleContext.ll000066400000000000000000000027701506646577400242670ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_debug_module %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-macosx %t.ll -o - -filetype=obj \ ; RUN: | llvm-dwarfdump -debug-info - | FileCheck %s ; CHECK: DW_TAG_module ; CHECK-NOT: NULL ; CHECK: DW_TAG_structure_type ; Hand-crafted based on ; struct s; ; struct s *s; source_filename = "test/DebugInfo/X86/DIModuleContext.ll" target triple = "spir64-unknown-unknown" %struct.s = type opaque @i = common addrspace(1) global %struct.s* null, align 8, !dbg !0 !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!11, !12} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "s", scope: !2, file: !3, line: 2, type: !9, isLocal: false, isDefinition: true) !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, imports: !6) !3 = !DIFile(filename: "test.c", directory: "/") !4 = !{} !5 = !{!0} !6 = !{!7} !7 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !2, entity: !8, file: !3, line: 11) !8 = !DIModule(scope: null, name: "Module", includePath: ".") !9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64, align: 64) !10 = !DICompositeType(tag: DW_TAG_structure_type, name: "s", scope: !8, file: !3, line: 1, flags: DIFlagFwdDecl) !11 = !{i32 2, !"Dwarf Version", i32 2} !12 = !{i32 2, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DW_AT_byte_size.ll000066400000000000000000000050311506646577400242240ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-darwin %t.ll -o %t -filetype=obj ; RUN: llvm-dwarfdump -all %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Checks that we don't emit a size for a pointer type. ; CHECK: DW_TAG_pointer_type ; CHECK-NEXT: DW_AT_type ; CHECK-NOT: DW_AT_byte_size ; CHECK: DW_TAG ; CHECK: .debug_info contents %struct.A = type { i32 } define i32 @_Z3fooP1A(%struct.A* %a) nounwind uwtable ssp !dbg !5 { entry: %a.addr = alloca %struct.A*, align 8 store %struct.A* %a, %struct.A** %a.addr, align 8 call void @llvm.dbg.declare(metadata %struct.A** %a.addr, metadata !16, metadata !DIExpression()), !dbg !17 %0 = load %struct.A*, %struct.A** %a.addr, align 8, !dbg !18 %b = getelementptr inbounds %struct.A, %struct.A* %0, i32 0, i32 0, !dbg !18 %1 = load i32, i32* %b, align 4, !dbg !18 ret i32 %1, !dbg !18 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!21} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.1 (trunk 150996)", isOptimized: false, emissionKind: FullDebug, file: !20, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !1 = !{} !5 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooP1A", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !20, scope: !6, type: !7) !6 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo") !7 = !DISubroutineType(types: !8) !8 = !{!9, !10} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11) !11 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 1, size: 32, align: 32, file: !20, elements: !12) !12 = !{!13} !13 = !DIDerivedType(tag: DW_TAG_member, name: "b", line: 1, size: 32, align: 32, file: !20, scope: !11, baseType: !9) !16 = !DILocalVariable(name: "a", line: 3, arg: 1, scope: !5, file: !6, type: !10) !17 = !DILocation(line: 3, column: 13, scope: !5) !18 = !DILocation(line: 4, column: 3, scope: !19) !19 = distinct !DILexicalBlock(line: 3, column: 16, file: !20, scope: !5) !20 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo") !21 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DW_AT_linkage_name.ll000066400000000000000000000122401506646577400246410ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-macosx %t.ll -o %t -filetype=obj ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s ; ; struct A { ; A(int i); ; ~A(); ; }; ; ; A::~A() {} ; ; void foo() { ; A a(1); ; } ; ; rdar://problem/16362674 ; ; Test that we do not emit a linkage name for the declaration of a destructor. ; Test that we do emit a linkage name for a specific instance of it. ; CHECK: DW_TAG_subprogram ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_name {{.*}} "~A" ; CHECK-NOT: DW_AT_MIPS_linkage_name ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_MIPS_linkage_name {{.*}} "_ZN1AD2Ev" ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_specification {{.*}} "~A" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" %struct.A = type { i8 } ; Function Attrs: nounwind ssp uwtable define void @_ZN1AD2Ev(%struct.A* %this) unnamed_addr #0 align 2 !dbg !17 { entry: %this.addr = alloca %struct.A*, align 8 store %struct.A* %this, %struct.A** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !26, metadata !DIExpression()), !dbg !28 %this1 = load %struct.A*, %struct.A** %this.addr ret void, !dbg !29 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 ; Function Attrs: nounwind ssp uwtable define void @_ZN1AD1Ev(%struct.A* %this) unnamed_addr #0 align 2 !dbg !18 { entry: %this.addr = alloca %struct.A*, align 8 store %struct.A* %this, %struct.A** %this.addr, align 8 call void @llvm.dbg.declare(metadata %struct.A** %this.addr, metadata !30, metadata !DIExpression()), !dbg !31 %this1 = load %struct.A*, %struct.A** %this.addr call void @_ZN1AD2Ev(%struct.A* %this1), !dbg !32 ret void, !dbg !33 } ; Function Attrs: ssp uwtable define void @_Z3foov() #2 !dbg !19 { entry: %a = alloca %struct.A, align 1 call void @llvm.dbg.declare(metadata %struct.A* %a, metadata !34, metadata !DIExpression()), !dbg !35 call void @_ZN1AC1Ei(%struct.A* %a, i32 1), !dbg !35 call void @_ZN1AD1Ev(%struct.A* %a), !dbg !36 ret void, !dbg !36 } declare void @_ZN1AC1Ei(%struct.A*, i32) attributes #0 = { nounwind ssp uwtable } attributes #1 = { nounwind readnone } attributes #2 = { ssp uwtable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!23, !24} !llvm.ident = !{!25} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "linkage-name.cpp", directory: "") !2 = !{} !3 = !{!4} !4 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", line: 1, size: 8, align: 8, file: !1, elements: !5, identifier: "_ZTS1A") !5 = !{!6, !12} !6 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !4, type: !7) !7 = !DISubroutineType(types: !8) !8 = !{null, !9, !10} !9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4) !10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !12 = !DISubprogram(name: "~A", line: 3, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 3, file: !1, scope: !4, type: !13) !13 = !DISubroutineType(types: !14) !14 = !{null, !9} !17 = distinct !DISubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: !4, type: !13, declaration: !12, retainedNodes: !2) !18 = distinct !DISubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: !4, type: !13, declaration: !12, retainedNodes: !2) !19 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 10, file: !1, scope: !20, type: !21, retainedNodes: !2) !20 = !DIFile(filename: "linkage-name.cpp", directory: "") !21 = !DISubroutineType(types: !22) !22 = !{null} !23 = !{i32 2, !"Dwarf Version", i32 2} !24 = !{i32 1, !"Debug Info Version", i32 3} !25 = !{!"clang version 3.5.0 "} !26 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !17, type: !27) !27 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !4) !28 = !DILocation(line: 0, scope: !17) !29 = !DILocation(line: 8, scope: !17) !30 = !DILocalVariable(name: "this", arg: 1, flags: DIFlagArtificial | DIFlagObjectPointer, scope: !18, type: !27) !31 = !DILocation(line: 0, scope: !18) !32 = !DILocation(line: 6, scope: !18) !33 = !DILocation(line: 8, scope: !18) !34 = !DILocalVariable(name: "a", line: 11, scope: !19, file: !20, type: !4) !35 = !DILocation(line: 11, scope: !19) !36 = !DILocation(line: 12, scope: !19) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DW_AT_specification.ll000066400000000000000000000050661506646577400250570ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-apple-darwin %t.ll -o %t -filetype=obj ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; test that the DW_AT_specification is a back edge in the file. ; CHECK: DW_TAG_subprogram ; CHECK-NEXT: DW_AT_linkage_name {{.*}} "_ZN3foo3barEv" ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_specification {{.*}} "_ZN3foo3barEv" source_filename = "test/DebugInfo/X86/DW_AT_specification.ll" @_ZZN3foo3barEvE1x = constant i32 0, align 4, !dbg !0 define void @_ZN3foo3barEv() !dbg !2 { entry: ret void, !dbg !17 } !llvm.dbg.cu = !{!8} !llvm.module.flags = !{!16} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 5, type: !14, isLocal: true, isDefinition: true) !2 = distinct !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", scope: null, file: !3, line: 4, type: !4, isLocal: false, isDefinition: true, scopeLine: 4, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !8, declaration: !11) !3 = !DIFile(filename: "nsNativeAppSupportBase.ii", directory: "/Users/espindola/mozilla-central/obj-x86_64-apple-darwin11.2.0/toolkit/library") !4 = !DISubroutineType(types: !5) !5 = !{null, !6} !6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64, align: 64, flags: DIFlagArtificial) !7 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !3, line: 1, flags: DIFlagFwdDecl) !8 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.0 ()", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !9, retainedTypes: !9, globals: !10, imports: !9) !9 = !{} !10 = !{!0} !11 = !DISubprogram(name: "bar", linkageName: "_ZN3foo3barEv", scope: !12, file: !3, line: 2, type: !4, isLocal: false, isDefinition: false, scopeLine: 2, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false) !12 = !DICompositeType(tag: DW_TAG_class_type, name: "foo", file: !3, line: 1, size: 8, align: 8, elements: !13) !13 = !{!11} !14 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !15) !15 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !16 = !{i32 1, !"Debug Info Version", i32 3} !17 = !DILocation(line: 6, column: 1, scope: !18) !18 = distinct !DILexicalBlock(scope: !2, file: !3, line: 4, column: 17) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/DW_AT_stmt_list_sec_offset.ll000066400000000000000000000040311506646577400264500ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=i686-w64-mingw32 -o %t -filetype=obj %t.ll ; RUN: llvm-dwarfdump -v -all %t | FileCheck %s ; RUN: llc -mtriple=i686-w64-mingw32 -o %t -filetype=obj -dwarf-version=3 %t.ll ; RUN: llvm-dwarfdump -v -all %t | FileCheck %s -check-prefix=DWARF3 ; CHECK: DW_AT_stmt_list [DW_FORM_sec_offset] ; DWARF3: DW_AT_stmt_list [DW_FORM_data4] ; ; generated from: ; clang -g -S -emit-llvm test.c -o test.ll ; int main() ; { ; return 0; ; } ; ModuleID = 'test.c' target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32" target triple = "spir64-unknown-unknown" ; Function Attrs: nounwind define i32 @main() #0 !dbg !4 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval ret i32 0, !dbg !10 } attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!9, !11} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "test.c", directory: "C:\5CProjects") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "test.c", directory: "C:CProjects") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{i32 2, !"Dwarf Version", i32 4} !10 = !DILocation(line: 3, scope: !4) !11 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/Fortran-DIModule.ll000066400000000000000000000035571506646577400243370ustar00rootroot00000000000000; This test checks attributes of a Fortran module. ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_debug_module %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-unknown-linux-gnu %t.ll -filetype=obj -o - | \ ; RUN: llvm-dwarfdump - | FileCheck %s ; CHECK: DW_TAG_module ; CHECK-NEXT: DW_AT_name ("dummy") ; CHECK-NEXT: DW_AT_decl_file ("/fortran{{[/\\]}}module.f90") ; CHECK-NEXT: DW_AT_decl_line (2) ; Generated from flang compiler, Fortran source to regenerate: ; module dummy ; integer :: foo ; end module dummy ; ModuleID = '/tmp/module-b198fa.ll' source_filename = "/tmp/module-b198fa.ll" target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" %struct_dummy_0_ = type <{ [4 x i8] }> @_dummy_0_ = common addrspace(1) global %struct_dummy_0_ zeroinitializer, align 64, !dbg !0 ; Function Attrs: noinline define float @dummy_() #0 { .L.entry: ret float undef } attributes #0 = { noinline "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" } !llvm.module.flags = !{!8, !9} !llvm.dbg.cu = !{!3} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = distinct !DIGlobalVariable(name: "foo", scope: !2, file: !4, type: !7, isLocal: false, isDefinition: true) !2 = !DIModule(scope: !3, name: "dummy", file: !4, line: 2) !3 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !4, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6, imports: !5) !4 = !DIFile(filename: "module.f90", directory: "/fortran") !5 = !{} !6 = !{!0} !7 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed) !8 = !{i32 2, !"Dwarf Version", i32 4} !9 = !{i32 2, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/InlinedFnLocalVar.ll000066400000000000000000000064441506646577400245540ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple i686-pc-cygwin -O2 %t.ll -o - | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Check struct X for dead variable xyz from inlined function foo. ; CHECK: Lsection_info ; CHECK: DW_TAG_structure_type ; CHECK-NEXT: info_string source_filename = "test/DebugInfo/X86/InlinedFnLocalVar.ll" @i = common global i32 0, !dbg !0 ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #0 ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #0 ; Function Attrs: nounwind ssp define i32 @bar() #1 !dbg !8 { entry: %0 = load i32, i32* @i, align 4, !dbg !11 tail call void @llvm.dbg.value(metadata i32 %0, metadata !13, metadata !24), !dbg !25 tail call void @llvm.dbg.declare(metadata !5, metadata !18, metadata !24), !dbg !26 %1 = mul nsw i32 %0, %0, !dbg !27 store i32 %1, i32* @i, align 4, !dbg !11 ret i32 %1, !dbg !28 } attributes #0 = { nounwind readnone } attributes #1 = { nounwind ssp } !llvm.dbg.cu = !{!4} !llvm.module.flags = !{!7} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "i", scope: !2, file: !2, line: 5, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "bar.c", directory: "/tmp/") !3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !4 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6, imports: !5) !5 = !{} !6 = !{!0} !7 = !{i32 1, !"Debug Info Version", i32 3} !8 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: !2, file: !2, line: 14, type: !9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !4) !9 = !DISubroutineType(types: !10) !10 = !{!3} !11 = !DILocation(line: 15, scope: !12) !12 = distinct !DILexicalBlock(scope: !8, file: !2, line: 14) !13 = !DILocalVariable(name: "j", arg: 1, scope: !14, file: !2, line: 9, type: !3) !14 = distinct !DISubprogram(name: "foo", scope: !2, file: !2, line: 9, type: !15, isLocal: true, isDefinition: true, scopeLine: 9, virtualIndex: 6, isOptimized: true, unit: !4, retainedNodes: !17) !15 = !DISubroutineType(types: !16) !16 = !{!3, !3} !17 = !{!13, !18} !18 = !DILocalVariable(name: "xyz", scope: !19, file: !2, line: 10, type: !20) !19 = distinct !DILexicalBlock(scope: !14, file: !2, line: 9) !20 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !14, file: !2, line: 10, size: 64, align: 32, elements: !21) !21 = !{!22, !23} !22 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !20, file: !2, line: 10, baseType: !3, size: 32, align: 32) !23 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !20, file: !2, line: 10, baseType: !3, size: 32, align: 32, offset: 32) !24 = !DIExpression() !25 = !DILocation(line: 9, scope: !14, inlinedAt: !11) !26 = !DILocation(line: 9, scope: !19, inlinedAt: !11) !27 = !DILocation(line: 11, scope: !19, inlinedAt: !11) !28 = !DILocation(line: 16, scope: !12) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/abstract_origin.ll000066400000000000000000000042301506646577400244230ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=obj %t.ll -o - | llvm-dwarfdump -debug-info - | FileCheck %s ; Generated at -O2 from: ; void f(); ; __attribute__((always_inline)) void g() { ; f(); ; } ; void h() { ; g(); ; }; ; CHECK: DW_TAG_subprogram ; CHECK: DW_AT_abstract_origin {{.*}}"g" ; CHECK-NOT: DW_AT_abstract_origin {{.*}}"g" ; CHECK: DW_TAG source_filename = "test.c" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: alwaysinline nounwind ssp uwtable define void @g() #0 !dbg !7 { entry: tail call void (...) @f() #3, !dbg !10 ret void, !dbg !11 } declare void @f(...) ; Function Attrs: nounwind ssp uwtable define void @h() #2 !dbg !12 { entry: tail call void (...) @f() #3, !dbg !13 ret void, !dbg !15 } attributes #0 = { alwaysinline nounwind ssp uwtable } attributes #2 = { nounwind ssp uwtable } attributes #3 = { nounwind } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "test.c", directory: "/Volumes/Data/llvm") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 2} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"PIC Level", i32 2} !6 = !{!"clang version 3.9.0 "} !7 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: true, unit: !0, retainedNodes: !2) !8 = !DISubroutineType(types: !9) !9 = !{null} !10 = !DILocation(line: 3, column: 3, scope: !7) !11 = !DILocation(line: 4, column: 1, scope: !7) !12 = distinct !DISubprogram(name: "h", scope: !1, file: !1, line: 5, type: !8, isLocal: false, isDefinition: true, scopeLine: 5, isOptimized: true, unit: !0, retainedNodes: !2) !13 = !DILocation(line: 3, column: 3, scope: !7, inlinedAt: !14) !14 = distinct !DILocation(line: 6, column: 3, scope: !12) !15 = !DILocation(line: 7, column: 1, scope: !12) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/aligned_stack_var.ll000066400000000000000000000043671506646577400247240ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc %t.ll -mtriple=x86_64-pc-linux-gnu -O0 -filetype=obj -o %t ; RUN: llvm-dwarfdump -debug-info %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; If stack is realigned, we shouldn't describe locations of local ; variables by giving offset from the frame pointer (%rbp): ; push %rpb ; mov %rsp,%rbp ; and ALIGNMENT,%rsp ; (%rsp and %rbp are different now) ; It's better to use offset from %rsp instead. ; DW_AT_location of variable "x" shouldn't be equal to ; (DW_OP_fbreg: .*): DW_OP_fbreg has code 0x91 ; CHECK: {{0x.* DW_TAG_variable}} ; CHECK-NOT: {{DW_AT_location.*DW_FORM_block1.*0x.*91}} ; CHECK: NULL define void @_Z3runv() nounwind uwtable !dbg !5 { entry: %x = alloca i32, align 32 call void @llvm.dbg.declare(metadata i32* %x, metadata !9, metadata !DIExpression()), !dbg !12 ret void, !dbg !13 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!15} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.2 (trunk 155696:155697) (llvm/trunk 155696)", isOptimized: false, emissionKind: FullDebug, file: !14, enums: !1, retainedTypes: !1, globals: !1, imports: !1) !1 = !{} !5 = distinct !DISubprogram(name: "run", linkageName: "_Z3runv", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !14, scope: !6, type: !7, retainedNodes: !1) !6 = !DIFile(filename: "test.cc", directory: "/home/samsonov/debuginfo") !7 = !DISubroutineType(types: !8) !8 = !{null} !9 = !DILocalVariable(name: "x", line: 2, scope: !10, file: !6, type: !11) !10 = distinct !DILexicalBlock(line: 1, column: 12, file: !14, scope: !5) !11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !12 = !DILocation(line: 2, column: 7, scope: !10) !13 = !DILocation(line: 3, column: 1, scope: !10) !14 = !DIFile(filename: "test.cc", directory: "/home/samsonov/debuginfo") !15 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/arguments.ll000066400000000000000000000073331506646577400232650ustar00rootroot00000000000000; REQUIRES: object-emission ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-unknown-unknown -O0 -filetype=obj < %t.ll > %t ; RUN: llvm-dwarfdump %t | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; IR generated from clang -g with the following source: ; struct foo { ; foo(const foo&); ; int i; ; }; ; ; void func(foo f, foo g) { ; f.i++; ; } ; CHECK: debug_info contents ; CHECK: DW_TAG_subprogram ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_linkage_name{{.*}}"_Z4func3fooS_" ; CHECK-NOT: NULL ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name{{.*}}"f" ; CHECK-NOT: NULL ; CHECK: DW_TAG_formal_parameter ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name{{.*}}"g" %struct.foo = type { i32 } ; Function Attrs: nounwind uwtable define void @_Z4func3fooS_(%struct.foo* %f, %struct.foo* %g) #0 !dbg !4 { entry: call void @llvm.dbg.declare(metadata %struct.foo* %f, metadata !19, metadata !DIExpression()), !dbg !20 call void @llvm.dbg.declare(metadata %struct.foo* %g, metadata !21, metadata !DIExpression()), !dbg !20 %i = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 0, i32 0, !dbg !22 %0 = load i32, i32* %i, align 4, !dbg !22 %inc = add nsw i32 %0, 1, !dbg !22 store i32 %inc, i32* %i, align 4, !dbg !22 ret void, !dbg !23 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!24} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch") !2 = !{} !4 = distinct !DISubprogram(name: "func", linkageName: "_Z4func3fooS_", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 6, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "scratch.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch") !6 = !DISubroutineType(types: !7) !7 = !{null, !8, !8} !8 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", line: 1, size: 32, align: 32, file: !1, elements: !9) !9 = !{!10, !12} !10 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 3, size: 32, align: 32, file: !1, scope: !8, baseType: !11) !11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !12 = !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 2, file: !1, scope: !8, type: !13) !13 = !DISubroutineType(types: !14) !14 = !{null, !15, !16} !15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !8) !16 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !17) !17 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8) !19 = !DILocalVariable(name: "f", line: 6, arg: 1, scope: !4, file: !5, type: !8) !20 = !DILocation(line: 6, scope: !4) !21 = !DILocalVariable(name: "g", line: 6, arg: 2, scope: !4, file: !5, type: !8) !22 = !DILocation(line: 7, scope: !4) !23 = !DILocation(line: 8, scope: !4) !24 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/coff_debug_info_type.ll000066400000000000000000000043201506646577400254100ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=i686-pc-mingw32 -accel-tables=Apple -filetype=asm -O0 < %t.ll | FileCheck %s ; RUN: llc -mtriple=i686-pc-cygwin -accel-tables=Apple -filetype=asm -O0 < %t.ll | FileCheck %s ; RUN: llc -mtriple=i686-w64-mingw32 -accel-tables=Apple -filetype=asm -O0 < %t.ll | FileCheck %s ; CHECK: .section .debug_info ; CHECK: .section .apple_names ; CHECK: .section .apple_types ; RUN: sed -e 's/"Dwarf Version"/"CodeView"/' %s \ ; RUN: | llc -mtriple=i686-pc-win32 -filetype=asm -O0 \ ; RUN: | FileCheck -check-prefix=WIN32 %s ; WIN32: .section .debug$S,"dr" ; RUN: llc -mtriple=i686-pc-win32 -filetype=null -O0 < %t.ll target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; generated from: ; clang -g -S -emit-llvm test.c -o test.ll ; int main() ; { ; return 0; ; } define i32 @main() #0 !dbg !4 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval ret i32 0, !dbg !10 } attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!9, !11} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "test.c", directory: "C:\5CProjects") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "test.c", directory: "C:CProjects") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{i32 2, !"Dwarf Version", i32 3} !10 = !DILocation(line: 3, scope: !4) !11 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/coff_relative_names.ll000066400000000000000000000033671506646577400252560ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=i686-w64-mingw32 -filetype=asm -O0 < %t.ll | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; CHECK: .secrel32 Linfo_string0 ; CHECK: .secrel32 Linfo_string1 ; ; generated from: ; clang -g -S -emit-llvm test.c -o test.ll ; int main() ; { ; return 0; ; } ; Function Attrs: nounwind define i32 @main() #0 !dbg !4 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval ret i32 0, !dbg !10 } attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!9, !11} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "test.c", directory: "C:\5CProjects") !2 = !{} !4 = distinct !DISubprogram(name: "main", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "test.c", directory: "C:CProjects") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{i32 2, !"Dwarf Version", i32 3} !10 = !DILocation(line: 3, scope: !4) !11 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/constant-aggregate.ll000066400000000000000000000120521506646577400250270ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple %t.ll -filetype=obj -o %t.o ; RUN: llvm-dwarfdump -v -debug-info %t.o | FileCheck %s ; Test emitting a constant for an aggregate type. ; ; clang -S -O1 -emit-llvm ; ; typedef struct { unsigned i; } S; ; ; unsigned foo(S s) { ; s.i = 1; ; return s.i; ; } ; ; class C { public: unsigned i; }; ; ; unsigned foo(C c) { ; c.i = 2; ; return c.i; ; } ; ; unsigned bar() { ; int a[1] = { 3 }; ; return a[0]; ; } ; ; CHECK: DW_TAG_formal_parameter ; CHECK-NEXT: DW_AT_const_value [DW_FORM_udata] (1) ; CHECK-NEXT: DW_AT_name {{.*}} "s" ; ; CHECK: DW_TAG_formal_parameter ; CHECK-NEXT: DW_AT_const_value [DW_FORM_udata] (2) ; CHECK-NEXT: DW_AT_name {{.*}} "c" ; ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_const_value [DW_FORM_udata] (3) ; CHECK-NEXT: DW_AT_name {{.*}} "a" ; ModuleID = 'sroasplit-4.cpp' target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: nounwind readnone ssp uwtable define i32 @_Z3foo1S(i32 %s.coerce) #0 !dbg !12 { entry: tail call void @llvm.dbg.value(metadata i32 %s.coerce, metadata !18, metadata !37), !dbg !38 tail call void @llvm.dbg.value(metadata i32 1, metadata !18, metadata !37), !dbg !38 ret i32 1, !dbg !39 } ; Function Attrs: nounwind readnone ssp uwtable define i32 @_Z3foo1C(i32 %c.coerce) #0 !dbg !19 { entry: tail call void @llvm.dbg.value(metadata i32 %c.coerce, metadata !23, metadata !37), !dbg !40 tail call void @llvm.dbg.value(metadata i32 2, metadata !23, metadata !37), !dbg !40 ret i32 2, !dbg !41 } ; Function Attrs: nounwind readnone ssp uwtable define i32 @_Z3barv() #0 !dbg !24 { entry: tail call void @llvm.dbg.value(metadata i32 3, metadata !28, metadata !37), !dbg !42 ret i32 3, !dbg !43 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { nounwind readnone ssp uwtable } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!33, !34, !35} !llvm.ident = !{!36} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 (trunk 225364) (llvm/trunk 225366)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2) !1 = !DIFile(filename: "sroasplit-4.cpp", directory: "") !2 = !{} !3 = !{!4, !8} !4 = !DICompositeType(tag: DW_TAG_structure_type, line: 1, size: 32, align: 32, file: !1, elements: !5, identifier: "_ZTS1S") !5 = !{!6} !6 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 1, size: 32, align: 32, file: !1, scope: !4, baseType: !7) !7 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned) !8 = !DICompositeType(tag: DW_TAG_class_type, name: "C", line: 8, size: 32, align: 32, file: !1, elements: !9, identifier: "_ZTS1C") !9 = !{!10} !10 = !DIDerivedType(tag: DW_TAG_member, name: "i", line: 8, size: 32, align: 32, flags: DIFlagPublic, file: !1, scope: !8, baseType: !7) !12 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foo1S", line: 3, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !1, scope: !13, type: !14, retainedNodes: !17) !13 = !DIFile(filename: "sroasplit-4.cpp", directory: "") !14 = !DISubroutineType(types: !15) !15 = !{!7, !16} !16 = !DIDerivedType(tag: DW_TAG_typedef, name: "S", line: 1, file: !1, baseType: !4) !17 = !{!18} !18 = !DILocalVariable(name: "s", line: 3, arg: 1, scope: !12, file: !13, type: !16) !19 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foo1C", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !13, type: !20, retainedNodes: !22) !20 = !DISubroutineType(types: !21) !21 = !{!7, !8} !22 = !{!23} !23 = !DILocalVariable(name: "c", line: 10, arg: 1, scope: !19, file: !13, type: !8) !24 = distinct !DISubprogram(name: "bar", linkageName: "_Z3barv", line: 15, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 15, file: !1, scope: !13, type: !25, retainedNodes: !27) !25 = !DISubroutineType(types: !26) !26 = !{!7} !27 = !{!28} !28 = !DILocalVariable(name: "a", line: 16, scope: !24, file: !13, type: !29) !29 = !DICompositeType(tag: DW_TAG_array_type, size: 32, align: 32, baseType: !30, elements: !31) !30 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !31 = !{!32} !32 = !DISubrange(count: 1) !33 = !{i32 2, !"Dwarf Version", i32 2} !34 = !{i32 2, !"Debug Info Version", i32 3} !35 = !{i32 1, !"PIC Level", i32 2} !36 = !{!"clang version 3.6.0 (trunk 225364) (llvm/trunk 225366)"} !37 = !DIExpression() !38 = !DILocation(line: 3, column: 16, scope: !12) !39 = !DILocation(line: 5, column: 3, scope: !12) !40 = !DILocation(line: 10, column: 16, scope: !19) !41 = !DILocation(line: 12, column: 3, scope: !19) !42 = !DILocation(line: 16, column: 6, scope: !24) !43 = !DILocation(line: 17, column: 3, scope: !24) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/constant-loclist.ll000066400000000000000000000071501506646577400245550ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -filetype=obj %t.ll -o - -experimental-debug-variable-locations=false | llvm-dwarfdump -v -debug-info - | FileCheck %s ; A hand-written testcase to check 64-bit constant handling in location lists. ; CHECK: .debug_info contents: ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_location [DW_FORM_data4] ( ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_constu 0x4000000000000000) ; CHECK-NEXT: DW_AT_name {{.*}}"d" ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_location [DW_FORM_data4] ( ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_consts +0 ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_consts +4611686018427387904) ; CHECK-NEXT: DW_AT_name {{.*}}"i" ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_location [DW_FORM_data4] ( ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_lit0 ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_constu 0x4000000000000000) ; CHECK-NEXT: DW_AT_name {{.*}}"u" source_filename = "test.c" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: nounwind ssp uwtable define void @main() #0 !dbg !7 { %1 = alloca double, align 8 %2 = alloca i64, align 8 %3 = alloca i64, align 8 store double 2.000000e+00, double* %1, align 8, !dbg !21 call void @llvm.dbg.value(metadata i64 0, metadata !22, metadata !15), !dbg !24 call void @llvm.dbg.value(metadata i64 0, metadata !25, metadata !15), !dbg !27 call void @llvm.dbg.value(metadata double 2.000000e+00, metadata !19, metadata !15), !dbg !21 store i64 4611686018427387904, i64* %2, align 8, !dbg !24 call void @llvm.dbg.value(metadata i64 4611686018427387904, metadata !22, metadata !15), !dbg !24 call void @llvm.dbg.value(metadata i64 4611686018427387904, metadata !25, metadata !15), !dbg !27 store i64 4611686018427387904, i64* %3, align 8, !dbg !27 ret void, !dbg !28 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 declare void @llvm.dbg.value(metadata, metadata, metadata) #1 attributes #0 = { nounwind ssp uwtable } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "test.c", directory: "/tmp") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 2} !4 = !{i32 2, !"Debug Info Version", i32 00000003} !5 = !{i32 1, !"PIC Level", i32 2} !6 = !{!"clang"} !7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !8 = !DISubroutineType(types: !{}) !15 = !DIExpression() !16 = !DILocation(line: 1, column: 14, scope: !7) !18 = !DILocation(line: 1, column: 24, scope: !7) !19 = !DILocalVariable(name: "d", scope: !7, file: !1, line: 2, type: !20) !20 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float) !21 = !DILocation(line: 2, column: 10, scope: !7) !22 = !DILocalVariable(name: "u", scope: !7, file: !1, line: 3, type: !23) !23 = !DIBasicType(name: "long long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned) !24 = !DILocation(line: 3, column: 22, scope: !7) !25 = !DILocalVariable(name: "i", scope: !7, file: !1, line: 4, type: !26) !26 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed) !27 = !DILocation(line: 4, column: 20, scope: !7) !28 = !DILocation(line: 5, column: 3, scope: !7) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/convert-debugloc.ll000066400000000000000000000075041506646577400245220ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-allow-extra-diexpressions ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -dwarf-version=5 -filetype=obj -O0 < %t.ll | llvm-dwarfdump - \ ; RUN: | FileCheck %s --check-prefix=DW5 "--implicit-check-not={{DW_TAG|NULL}}" ; RUN: llc -mtriple=%triple -dwarf-version=4 -filetype=obj -O0 < %t.ll | llvm-dwarfdump - \ ; RUN: | FileCheck %s --check-prefix=DW4 "--implicit-check-not={{DW_TAG|NULL}}" ; DW5: .debug_info contents: ; DW5: DW_TAG_compile_unit ; DW5:[[SIG8:.*]]: DW_TAG_base_type ; DW5-NEXT:DW_AT_name ("DW_ATE_signed_8") ; DW5-NEXT:DW_AT_encoding (DW_ATE_signed) ; DW5-NEXT:DW_AT_byte_size (0x01) ; DW5-NOT: DW_AT ; DW5:[[SIG32:.*]]: DW_TAG_base_type ; DW5-NEXT:DW_AT_name ("DW_ATE_signed_32") ; DW5-NEXT:DW_AT_encoding (DW_ATE_signed) ; DW5-NEXT:DW_AT_byte_size (0x04) ; DW5-NOT: DW_AT ; DW5: DW_TAG_subprogram ; DW5: DW_TAG_formal_parameter ; DW5: DW_TAG_variable ; DW5: DW_AT_location ( ; DW5: {{.*}}, DW_OP_convert ([[SIG8]]) "DW_ATE_signed_8", DW_OP_convert ([[SIG32]]) "DW_ATE_signed_32", DW_OP_stack_value) ; DW5: DW_AT_name ("y") ; DW5: NULL ; DW5: DW_TAG_base_type ; DW5: DW_AT_name ("signed char") ; DW5: DW_TAG_base_type ; DW5: DW_AT_name ("int") ; DW5: NULL ; DW4: .debug_info contents: ; DW4: DW_TAG_compile_unit ; DW4: DW_TAG_subprogram ; DW4: DW_TAG_formal_parameter ; DW4: DW_TAG_variable ; DW4: DW_AT_location ( ; DW4: {{.*}}, DW_OP_dup, DW_OP_constu 0x7, DW_OP_shr, DW_OP_lit0, DW_OP_not, DW_OP_mul, DW_OP_constu 0x8, DW_OP_shl, DW_OP_or, DW_OP_stack_value) ; DW4: DW_AT_name ("y") ; DW4: NULL ; DW4: DW_TAG_base_type ; DW4: DW_AT_name ("signed char") ; DW4: DW_TAG_base_type ; DW4: DW_AT_name ("int") ; DW4: NULL target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: noinline nounwind uwtable define dso_local signext i8 @foo(i8 signext %x) !dbg !7 { entry: call void @llvm.dbg.value(metadata i8 %x, metadata !11, metadata !DIExpression()), !dbg !12 call void @llvm.dbg.value(metadata i8 32, metadata !13, metadata !DIExpression(DW_OP_lit0, DW_OP_plus, DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value)), !dbg !15 ret i8 %x, !dbg !16 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.declare(metadata, metadata, metadata) ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.value(metadata, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (trunk 353791) (llvm/trunk 353801)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) !1 = !DIFile(filename: "dbg.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "2a034da6937f5b9cf6dd2d89127f57fd") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 5} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{!"clang version 9.0.0 (trunk 353791) (llvm/trunk 353801)"} !7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) !8 = !DISubroutineType(types: !9) !9 = !{!10, !10} !10 = !DIBasicType(name: "signed char", size: 8, encoding: DW_ATE_signed_char) !11 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 1, type: !10) !12 = !DILocation(line: 1, column: 29, scope: !7) !13 = !DILocalVariable(name: "y", scope: !7, file: !1, line: 3, type: !14) !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !DILocation(line: 3, column: 14, scope: !7) !16 = !DILocation(line: 4, column: 3, scope: !7) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/cu-ranges.ll000066400000000000000000000075101506646577400231410ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -split-dwarf-file=foo.dwo -O0 %t.ll -function-sections -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t ; RUN: llvm-dwarfdump -debug-abbrev %t | FileCheck --check-prefix=FUNCTION-SECTIONS %s ; RUN: llvm-readobj --relocations %t | FileCheck --check-prefix=FUNCTION-SECTIONS-RELOCS %s ; RUN: llc -split-dwarf-file=foo.dwo -O0 %t.ll -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t ; RUN: llvm-dwarfdump -debug-abbrev %t | FileCheck --check-prefix=NO-FUNCTION-SECTIONS %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; From: ; int foo (int a) { ; return a+1; ; } ; int bar (int b) { ; return b+2; ; } ; With function sections enabled make sure that we have a DW_AT_ranges attribute. ; FUNCTION-SECTIONS: DW_AT_ranges ; Check that we have a relocation against the .debug_ranges section. ; FUNCTION-SECTIONS-RELOCS: R_X86_64_32 .debug_ranges 0x0 ; Without function sections enabled make sure that we have no DW_AT_ranges attribute. ; NO-FUNCTION-SECTIONS-NOT: DW_AT_ranges ; NO-FUNCTION-SECTIONS: DW_AT_low_pc DW_FORM_addr ; NO-FUNCTION-SECTIONS-NOT: DW_AT_ranges ; Function Attrs: nounwind uwtable define i32 @foo(i32 %a) #0 !dbg !4 { entry: %a.addr = alloca i32, align 4 store i32 %a, i32* %a.addr, align 4 call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !13, metadata !DIExpression()), !dbg !14 %0 = load i32, i32* %a.addr, align 4, !dbg !14 %add = add nsw i32 %0, 1, !dbg !14 ret i32 %add, !dbg !14 } ; Function Attrs: nounwind readnone declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 ; Function Attrs: nounwind uwtable define i32 @bar(i32 %b) #0 !dbg !9 { entry: %b.addr = alloca i32, align 4 store i32 %b, i32* %b.addr, align 4 call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !15, metadata !DIExpression()), !dbg !16 %0 = load i32, i32* %b.addr, align 4, !dbg !16 %add = add nsw i32 %0, 2, !dbg !16 ret i32 %add, !dbg !16 } attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!10, !11} !llvm.ident = !{!12} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo") !2 = !{} !4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2) !5 = !DIFile(filename: "z.c", directory: "/usr/local/google/home/echristo") !6 = !DISubroutineType(types: !7) !7 = !{!8, !8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = distinct !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2) !10 = !{i32 2, !"Dwarf Version", i32 4} !11 = !{i32 1, !"Debug Info Version", i32 3} !12 = !{!"clang version 3.5.0 (trunk 204164) (llvm/trunk 204183)"} !13 = !DILocalVariable(name: "a", line: 1, arg: 1, scope: !4, file: !5, type: !8) !14 = !DILocation(line: 1, scope: !4) !15 = !DILocalVariable(name: "b", line: 2, arg: 1, scope: !9, file: !5, type: !8) !16 = !DILocation(line: 2, scope: !9) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/data_member_location.ll000066400000000000000000000047221506646577400254070ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-linux -O0 -o - -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info -| FileCheck %s ; RUN: llc -mtriple=x86_64-linux -dwarf-version=2 -O0 -o - -filetype=obj < %t.ll | llvm-dwarfdump -v -debug-info -| FileCheck -check-prefix=DWARF2 %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Generated from Clang with the following source: ; ; struct foo { ; char c; ; int i; ; }; ; ; foo f; ; CHECK: DW_AT_name {{.*}} "c" ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_data_member_location {{.*}} (0x00) ; CHECK: DW_AT_name {{.*}} "i" ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_data_member_location {{.*}} (0x04) ; DWARF2: DW_AT_name {{.*}} "c" ; DWARF2-NOT: DW_TAG ; DWARF2: DW_AT_data_member_location {{.*}} (DW_OP_plus_uconst 0x0) ; DWARF2: DW_AT_name {{.*}} "i" ; DWARF2-NOT: DW_TAG ; DWARF2: DW_AT_data_member_location {{.*}} (DW_OP_plus_uconst 0x4) source_filename = "test/DebugInfo/X86/data_member_location.ll" %struct.foo = type { i8, i32 } @f = global %struct.foo zeroinitializer, align 4, !dbg !0 !llvm.dbg.cu = !{!9} !llvm.module.flags = !{!13, !14} !llvm.ident = !{!15} !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression()) !1 = !DIGlobalVariable(name: "f", scope: null, file: !2, line: 6, type: !3, isLocal: false, isDefinition: true) !2 = !DIFile(filename: "data_member_location.cpp", directory: "/tmp/dbginfo") !3 = !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !2, line: 1, size: 64, align: 32, elements: !4, identifier: "_ZTS3foo") !4 = !{!5, !7} !5 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !3, file: !2, line: 2, baseType: !6, size: 8, align: 8) !6 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !7 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !3, file: !2, line: 3, baseType: !8, size: 32, align: 32, offset: 32) !8 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang version 3.4 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !10, retainedTypes: !11, globals: !12, imports: !10) !10 = !{} !11 = !{!3} !12 = !{!0} !13 = !{i32 2, !"Dwarf Version", i32 4} !14 = !{i32 1, !"Debug Info Version", i32 3} !15 = !{!"clang version 3.4 "} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-byval-parameter.ll000066400000000000000000000065621506646577400251100ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -march=x86 -asm-verbose < %t.ll | grep DW_TAG_formal_parameter target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" %struct.Pt = type { double, double } %struct.Rect = type { %struct.Pt, %struct.Pt } define double @foo(%struct.Rect* byval(%struct.Rect) %my_r0) nounwind ssp !dbg !1 { entry: %retval = alloca double ; [#uses=2] %0 = alloca double ; [#uses=2] %"alloca point" = bitcast i32 0 to i32 ; [#uses=0] call void @llvm.dbg.declare(metadata %struct.Rect* %my_r0, metadata !0, metadata !DIExpression()), !dbg !15 %1 = getelementptr inbounds %struct.Rect, %struct.Rect* %my_r0, i32 0, i32 0, !dbg !16 ; <%struct.Pt*> [#uses=1] %2 = getelementptr inbounds %struct.Pt, %struct.Pt* %1, i32 0, i32 0, !dbg !16 ; [#uses=1] %3 = load double, double* %2, align 8, !dbg !16 ; [#uses=1] store double %3, double* %0, align 8, !dbg !16 %4 = load double, double* %0, align 8, !dbg !16 ; [#uses=1] store double %4, double* %retval, align 8, !dbg !16 br label %return, !dbg !16 return: ; preds = %entry %retval1 = load double, double* %retval, !dbg !16 ; [#uses=1] ret double %retval1, !dbg !16 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone !llvm.dbg.cu = !{!3} !llvm.module.flags = !{!21} !0 = !DILocalVariable(name: "my_r0", line: 11, arg: 1, scope: !1, file: !2, type: !7) !1 = distinct !DISubprogram(name: "foo", linkageName: "foo", line: 11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, file: !19, scope: !2, type: !4) !2 = !DIFile(filename: "b2.c", directory: "/tmp/") !3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: false, emissionKind: FullDebug, file: !19, enums: !20, retainedTypes: !20, imports: null) !4 = !DISubroutineType(types: !5) !5 = !{!6, !7} !6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float) !7 = !DICompositeType(tag: DW_TAG_structure_type, name: "Rect", line: 6, size: 256, align: 64, file: !19, scope: !2, elements: !8) !8 = !{!9, !14} !9 = !DIDerivedType(tag: DW_TAG_member, name: "P1", line: 7, size: 128, align: 64, file: !19, scope: !7, baseType: !10) !10 = !DICompositeType(tag: DW_TAG_structure_type, name: "Pt", line: 1, size: 128, align: 64, file: !19, scope: !2, elements: !11) !11 = !{!12, !13} !12 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 64, align: 64, file: !19, scope: !10, baseType: !6) !13 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 3, size: 64, align: 64, offset: 64, file: !19, scope: !10, baseType: !6) !14 = !DIDerivedType(tag: DW_TAG_member, name: "P2", line: 8, size: 128, align: 64, offset: 128, file: !19, scope: !7, baseType: !10) !15 = !DILocation(line: 11, scope: !1) !16 = !DILocation(line: 12, scope: !17) !17 = distinct !DILexicalBlock(line: 11, column: 0, file: !19, scope: !1) !19 = !DIFile(filename: "b2.c", directory: "/tmp/") !20 = !{} !21 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-declare-alloca.ll000066400000000000000000000053171506646577400246420ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll | FileCheck %s ; RUN: llc -mtriple=%triple < %t.ll -filetype=obj | llvm-dwarfdump -v - --debug-info | FileCheck %s --check-prefix=DWARF ; This should use the frame index side table for allocas, not DBG_VALUE ; instructions. For SDAG ISel, this test would see an SDNode materializing the ; argument to escape_foo and we'd get DBG_VALUE MachineInstr. ; CHECK-LABEL: use_dbg_declare: ; CHECK-NOT: #DEBUG_VALUE ; DWARF: DW_TAG_variable ; DWARF-NEXT: DW_AT_location [DW_FORM_exprloc] (DW_OP_fbreg +0) ; DWARF-NEXT: DW_AT_name [DW_FORM_strp] ( {{.*}} = "o") ; ModuleID = 't.c' source_filename = "t.c" target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" %struct.Foo = type { i32 } ; Function Attrs: noinline nounwind uwtable define void @use_dbg_declare() #0 !dbg !7 { entry: %o = alloca %struct.Foo, align 4 call void @llvm.dbg.declare(metadata %struct.Foo* %o, metadata !10, metadata !15), !dbg !16 call void @escape_foo(%struct.Foo* %o), !dbg !17 ret void, !dbg !18 } ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 declare void @escape_foo(%struct.Foo*) attributes #0 = { noinline nounwind uwtable } attributes #1 = { nounwind readnone speculatable } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!3, !4, !5} !llvm.ident = !{!6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) !1 = !DIFile(filename: "t.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild") !2 = !{} !3 = !{i32 2, !"Dwarf Version", i32 4} !4 = !{i32 2, !"Debug Info Version", i32 3} !5 = !{i32 1, !"wchar_size", i32 4} !6 = !{!"clang version 6.0.0 "} !7 = distinct !DISubprogram(name: "use_dbg_declare", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2) !8 = !DISubroutineType(types: !9) !9 = !{null} !10 = !DILocalVariable(name: "o", scope: !7, file: !1, line: 4, type: !11) !11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: !1, line: 1, size: 32, elements: !12) !12 = !{!13} !13 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !11, file: !1, line: 1, baseType: !14, size: 32) !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) !15 = !DIExpression() !16 = !DILocation(line: 4, column: 14, scope: !7) !17 = !DILocation(line: 5, column: 3, scope: !7) !18 = !DILocation(line: 6, column: 1, scope: !7) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-declare-arg.ll000066400000000000000000000176201506646577400241600ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 -fast-isel=true -filetype=obj -o - %t.ll | llvm-dwarfdump -v - | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "spir64-unknown-unknown" ; rdar://problem/9321650 ; C++ source: ; class A { public: int x; int y; int z; int o; ~A() { x = 1; }}; ; ; A foo(int i) { ; int j = 0; ; if (i == 42) { ; j = i + 1; ; }; ; A my_a; ; my_a.x = j; ; return my_a; ; } ; CHECK: DW_AT_name {{.*}}"j" ; CHECK: DW_TAG_variable ; CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset] ( ; CHECK-NEXT: 0x{{.*}}, 0x{{.*}}: DW_OP_breg7 RSP+8, DW_OP_deref) ; CHECK-NEXT: DW_AT_name {{.*}}"my_a" %class.A = type { i32, i32, i32, i32 } define void @_Z3fooi(%class.A* sret(%class.A) %agg.result, i32 %i) ssp !dbg !19 { entry: %i.addr = alloca i32, align 4 %j = alloca i32, align 4 %nrvo = alloca i1 %cleanup.dest.slot = alloca i32 store i32 %i, i32* %i.addr, align 4 call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !26, metadata !DIExpression()), !dbg !27 call void @llvm.dbg.declare(metadata i32* %j, metadata !28, metadata !DIExpression()), !dbg !30 store i32 0, i32* %j, align 4, !dbg !31 %tmp = load i32, i32* %i.addr, align 4, !dbg !32 %cmp = icmp eq i32 %tmp, 42, !dbg !32 br i1 %cmp, label %if.then, label %if.end, !dbg !32 if.then: ; preds = %entry %tmp1 = load i32, i32* %i.addr, align 4, !dbg !33 %add = add nsw i32 %tmp1, 1, !dbg !33 store i32 %add, i32* %j, align 4, !dbg !33 br label %if.end, !dbg !35 if.end: ; preds = %if.then, %entry store i1 false, i1* %nrvo, !dbg !36 call void @llvm.dbg.declare(metadata %class.A* %agg.result, metadata !37, metadata !DIExpression()), !dbg !39 %tmp2 = load i32, i32* %j, align 4, !dbg !40 %x = getelementptr inbounds %class.A, %class.A* %agg.result, i32 0, i32 0, !dbg !40 store i32 %tmp2, i32* %x, align 4, !dbg !40 store i1 true, i1* %nrvo, !dbg !41 store i32 1, i32* %cleanup.dest.slot %nrvo.val = load i1, i1* %nrvo, !dbg !42 br i1 %nrvo.val, label %nrvo.skipdtor, label %nrvo.unused, !dbg !42 nrvo.unused: ; preds = %if.end call void @_ZN1AD1Ev(%class.A* %agg.result), !dbg !42 br label %nrvo.skipdtor, !dbg !42 nrvo.skipdtor: ; preds = %nrvo.unused, %if.end ret void, !dbg !42 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone define linkonce_odr void @_ZN1AD1Ev(%class.A* %this) unnamed_addr ssp align 2 !dbg !22 { entry: %this.addr = alloca %class.A*, align 8 store %class.A* %this, %class.A** %this.addr, align 8 call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !43, metadata !DIExpression()), !dbg !44 %this1 = load %class.A*, %class.A** %this.addr call void @_ZN1AD2Ev(%class.A* %this1), !dbg !53 ret void, !dbg !45 } define linkonce_odr void @_ZN1AD2Ev(%class.A* %this) unnamed_addr nounwind ssp align 2 !dbg !25 { entry: %this.addr = alloca %class.A*, align 8 store %class.A* %this, %class.A** %this.addr, align 8 call void @llvm.dbg.declare(metadata %class.A** %this.addr, metadata !46, metadata !DIExpression()), !dbg !47 %this1 = load %class.A*, %class.A** %this.addr %x = getelementptr inbounds %class.A, %class.A* %this1, i32 0, i32 0, !dbg !48 store i32 1, i32* %x, align 4, !dbg !48 ret void, !dbg !48 } !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!52} !0 = !DISubprogram(name: "~A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11) !1 = !DICompositeType(tag: DW_TAG_class_type, name: "A", line: 2, size: 128, align: 32, file: !51, scope: !2, elements: !4) !2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.0 (trunk 130127)", isOptimized: false, emissionKind: FullDebug, file: !51, enums: !{}, retainedTypes: !{}) !3 = !DIFile(filename: "a.cc", directory: "/private/tmp") !4 = !{!5, !7, !8, !9, !0, !10, !14} !5 = !DIDerivedType(tag: DW_TAG_member, name: "x", line: 2, size: 32, align: 32, file: !51, scope: !3, baseType: !6) !6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !7 = !DIDerivedType(tag: DW_TAG_member, name: "y", line: 2, size: 32, align: 32, offset: 32, file: !51, scope: !3, baseType: !6) !8 = !DIDerivedType(tag: DW_TAG_member, name: "z", line: 2, size: 32, align: 32, offset: 64, file: !51, scope: !3, baseType: !6) !9 = !DIDerivedType(tag: DW_TAG_member, name: "o", line: 2, size: 32, align: 32, offset: 96, file: !51, scope: !3, baseType: !6) !10 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !11) !11 = !DISubroutineType(types: !12) !12 = !{null, !13} !13 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial, file: !3, baseType: !1) !14 = !DISubprogram(name: "A", line: 2, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, file: !51, scope: !1, type: !15) !15 = !DISubroutineType(types: !16) !16 = !{null, !13, !17} !17 = !DIDerivedType(tag: DW_TAG_reference_type, scope: !2, baseType: !18) !18 = !DIDerivedType(tag: DW_TAG_const_type, file: !3, baseType: !1) !19 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !51, scope: !3, type: !20) !20 = !DISubroutineType(types: !21) !21 = !{!1} !22 = distinct !DISubprogram(name: "~A", linkageName: "_ZN1AD1Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !51, scope: !3, type: !23) !23 = !DISubroutineType(types: !24) !24 = !{null} !25 = distinct !DISubprogram(name: "~A", linkageName: "_ZN1AD2Ev", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !51, scope: !3, type: !23) !26 = !DILocalVariable(name: "i", line: 4, arg: 1, scope: !19, file: !3, type: !6) !27 = !DILocation(line: 4, column: 11, scope: !19) !28 = !DILocalVariable(name: "j", line: 5, scope: !29, file: !3, type: !6) !29 = distinct !DILexicalBlock(line: 4, column: 14, file: !51, scope: !19) !30 = !DILocation(line: 5, column: 7, scope: !29) !31 = !DILocation(line: 5, column: 12, scope: !29) !32 = !DILocation(line: 6, column: 3, scope: !29) !33 = !DILocation(line: 7, column: 5, scope: !34) !34 = distinct !DILexicalBlock(line: 6, column: 16, file: !51, scope: !29) !35 = !DILocation(line: 8, column: 3, scope: !34) !36 = !DILocation(line: 9, column: 9, scope: !29) !37 = !DILocalVariable(name: "my_a", line: 9, scope: !29, file: !3, type: !38) !38 = !DIDerivedType(tag: DW_TAG_reference_type, file: !3, baseType: !1) !39 = !DILocation(line: 9, column: 5, scope: !29) !40 = !DILocation(line: 10, column: 3, scope: !29) !41 = !DILocation(line: 11, column: 3, scope: !29) !42 = !DILocation(line: 12, column: 1, scope: !29) !43 = !DILocalVariable(name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !22, file: !3, type: !13) !44 = !DILocation(line: 2, column: 47, scope: !22) !45 = !DILocation(line: 2, column: 61, scope: !22) !46 = !DILocalVariable(name: "this", line: 2, arg: 1, flags: DIFlagArtificial, scope: !25, file: !3, type: !13) !47 = !DILocation(line: 2, column: 47, scope: !25) !48 = !DILocation(line: 2, column: 54, scope: !49) !49 = distinct !DILexicalBlock(line: 2, column: 52, file: !51, scope: !25) !51 = !DIFile(filename: "a.cc", directory: "/private/tmp") !52 = !{i32 1, !"Debug Info Version", i32 3} !53 = !DILocation(line: 0, scope: !22) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-declare.ll000066400000000000000000000067461506646577400234200ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_INTEL_variable_length_array ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc < %t.ll -O0 -mtriple x86_64-apple-darwin | FileCheck %s ; RUN: llc < %t.ll -O0 -mtriple x86_64-apple-darwin -filetype=obj \ ; RUN: | llvm-dwarfdump -v - --debug-info | FileCheck %s --check-prefix=DWARF target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; ; CHECK-LABEL: _foo: ; CHECK-NOT: #DEBUG_VALUE ; "[DW_FORM_exprloc] <0x2> 91 XX" means fbreg uleb(XX) ; DWARF-LABEL: DW_TAG_formal_parameter ; DWARF-NEXT: DW_AT_location [DW_FORM_exprloc] (DW_OP_fbreg -8) ; DWARF-NEXT: DW_AT_name [DW_FORM_strp] ( {{.*}} = "x") ; FIXME: There is no debug info to describe "a". define i32 @foo(i32* %x) nounwind uwtable ssp !dbg !5 { entry: %x.addr = alloca i32*, align 8 %saved_stack = alloca i8* %cleanup.dest.slot = alloca i32 store i32* %x, i32** %x.addr, align 8 call void @llvm.dbg.declare(metadata i32** %x.addr, metadata !14, metadata !DIExpression()), !dbg !15 %0 = load i32*, i32** %x.addr, align 8, !dbg !16 %1 = load i32, i32* %0, align 4, !dbg !16 %2 = zext i32 %1 to i64, !dbg !16 %3 = call i8* @llvm.stacksave(), !dbg !16 store i8* %3, i8** %saved_stack, !dbg !16 %vla = alloca i8, i64 %2, align 16, !dbg !16 call void @llvm.dbg.declare(metadata i8* %vla, metadata !18, metadata !DIExpression()), !dbg !23 store i32 1, i32* %cleanup.dest.slot %4 = load i8*, i8** %saved_stack, !dbg !24 call void @llvm.stackrestore(i8* %4), !dbg !24 ret i32 0, !dbg !25 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone declare i8* @llvm.stacksave() nounwind declare void @llvm.stackrestore(i8*) nounwind !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!27} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 153698)", isOptimized: false, emissionKind: FullDebug, file: !26, enums: !1, retainedTypes: !1, globals: !1) !1 = !{} !5 = distinct !DISubprogram(name: "foo", line: 6, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, file: !26, scope: !0, type: !7) !6 = !DIFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm") !7 = !DISubroutineType(types: !8) !8 = !{!9, !10} !9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11) !11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !9) !14 = !DILocalVariable(name: "x", line: 5, arg: 1, scope: !5, file: !6, type: !10) !15 = !DILocation(line: 5, column: 21, scope: !5) !16 = !DILocation(line: 7, column: 13, scope: !17) !17 = distinct !DILexicalBlock(line: 6, column: 1, file: !26, scope: !5) !18 = !DILocalVariable(name: "a", line: 7, scope: !17, file: !6, type: !19) !19 = !DICompositeType(tag: DW_TAG_array_type, align: 8, baseType: !20, elements: !21) !20 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char) !21 = !{!22} !22 = !DISubrange(count: -1) !23 = !DILocation(line: 7, column: 8, scope: !17) !24 = !DILocation(line: 9, column: 1, scope: !17) !25 = !DILocation(line: 8, column: 3, scope: !17) !26 = !DIFile(filename: "20020104-2.c", directory: "/Volumes/Sandbox/llvm") !27 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-file-name.ll000066400000000000000000000024771506646577400236530ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple x86_64-apple-darwin10.0.0 < %t.ll | FileCheck %s target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" ; Verify that the file name is relative to the directory. ; rdar://problem/8884898 ; CHECK: file 1 "/Users/manav/one/two" "simple.c" declare i32 @printf(i8*, ...) nounwind define i32 @main() nounwind !dbg !6 { ret i32 0 } !llvm.dbg.cu = !{!2} !llvm.module.flags = !{!12} !1 = !DIFile(filename: "simple.c", directory: "/Users/manav/one/two") !2 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "LLVM build 00", isOptimized: true, emissionKind: FullDebug, file: !10, enums: !11, retainedTypes: !11) !5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !6 = distinct !DISubprogram(name: "main", linkageName: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !10, scope: !1, type: !7) !7 = !DISubroutineType(types: !8) !8 = !{!5} !10 = !DIFile(filename: "simple.c", directory: "/Users/manav/one/two") !11 = !{} !12 = !{i32 1, !"Debug Info Version", i32 3} SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-prolog-end.ll000066400000000000000000000062211506646577400240530ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O0 < %t.ll | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "spir64-unknown-unknown" ;CHECK-LABEL: foo: ;CHECK: .loc 1 2 11 prologue_end define i32 @foo(i32 %i) nounwind ssp !dbg !1 { entry: %i.addr = alloca i32, align 4 %j = alloca i32, align 4 store i32 %i, i32* %i.addr, align 4 call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !7, metadata !DIExpression()), !dbg !8 call void @llvm.dbg.declare(metadata i32* %j, metadata !9, metadata !DIExpression()), !dbg !11 store i32 2, i32* %j, align 4, !dbg !12 %tmp = load i32, i32* %j, align 4, !dbg !13 %inc = add nsw i32 %tmp, 1, !dbg !13 store i32 %inc, i32* %j, align 4, !dbg !13 %tmp1 = load i32, i32* %j, align 4, !dbg !14 %tmp2 = load i32, i32* %i.addr, align 4, !dbg !14 %add = add nsw i32 %tmp1, %tmp2, !dbg !14 store i32 %add, i32* %j, align 4, !dbg !14 %tmp3 = load i32, i32* %j, align 4, !dbg !15 ret i32 %tmp3, !dbg !15 } declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone ;CHECK-LABEL: main: ;CHECK: .loc 1 8 2 prologue_end define i32 @main() nounwind ssp !dbg !6 { entry: %retval = alloca i32, align 4 store i32 0, i32* %retval, !dbg !22 %call = call i32 @foo(i32 21), !dbg !16 ret i32 %call, !dbg !16 } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!21} !18 = !{!1, !6} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 131100)", isOptimized: false, emissionKind: FullDebug, file: !19, enums: !20, retainedTypes: !20, imports: null) !1 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !19, scope: !2, type: !3) !2 = !DIFile(filename: "/tmp/a.c", directory: "/private/tmp") !3 = !DISubroutineType(types: !4) !4 = !{!5} !5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !6 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !0, scopeLine: 7, file: !19, scope: !2, type: !3) !7 = !DILocalVariable(name: "i", line: 1, arg: 1, scope: !1, file: !2, type: !5) !8 = !DILocation(line: 1, column: 13, scope: !1) !9 = !DILocalVariable(name: "j", line: 2, scope: !10, file: !2, type: !5) !10 = distinct !DILexicalBlock(line: 1, column: 16, file: !19, scope: !1) !11 = !DILocation(line: 2, column: 6, scope: !10) !12 = !DILocation(line: 2, column: 11, scope: !10) !13 = !DILocation(line: 3, column: 2, scope: !10) !14 = !DILocation(line: 4, column: 2, scope: !10) !15 = !DILocation(line: 5, column: 2, scope: !10) !16 = !DILocation(line: 8, column: 2, scope: !17) !17 = distinct !DILexicalBlock(line: 7, column: 12, file: !19, scope: !6) !19 = !DIFile(filename: "/tmp/a.c", directory: "/private/tmp") !20 = !{} !21 = !{i32 1, !"Debug Info Version", i32 3} !22 = !DILocation(line: 0, column: 0, scope: !17) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-value-const-byref.ll000066400000000000000000000064621506646577400253610ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple -O1 -filetype=obj -o - %t.ll | llvm-dwarfdump -all - | FileCheck %s ; Generated with -O1 from: ; int f1(); ; void f2(int*); ; int f3(int); ; ; int foo() { ; int i = 3; ; f3(i); ; i = 7; ; i = f1(); ; f2(&i); ; return 0; ; } ; ; Test that we generate valid debug info for optimized code, ; particularly variables that are described as constants and passed ; by reference. ; rdar://problem/14874886 ; ; CHECK: .debug_info contents: ; CHECK: DW_TAG_variable ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_location {{.*}}({{.*}} ; CHECK-NEXT: 0x{{0*.*}}, [[C1:0x.*]]): DW_OP_consts +3 ; CHECK-NEXT: [[C1]], [[C2:0x.*]]): DW_OP_consts +7 ; CHECK-NEXT: [[C2]], [[R1:0x.*]]): DW_OP_reg0 RAX ; CHECK-NEXT: [[R1]], [[R2:0x.*]]): DW_OP_breg7 RSP+4) ; CHECK-NOT: DW_TAG ; CHECK: DW_AT_name{{.*}}"i" target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "spir64-unknown-unknown" ; Function Attrs: nounwind ssp uwtable define i32 @foo() #0 !dbg !4 { entry: %i = alloca i32, align 4 call void @llvm.dbg.value(metadata i32 3, metadata !10, metadata !DIExpression()), !dbg !15 %call = call i32 @f3(i32 3) #3, !dbg !16 call void @llvm.dbg.value(metadata i32 7, metadata !10, metadata !DIExpression()), !dbg !18 %call1 = call i32 (...) @f1() #3, !dbg !19 call void @llvm.dbg.value(metadata i32 %call1, metadata !10, metadata !DIExpression()), !dbg !19 store i32 %call1, i32* %i, align 4, !dbg !19, !tbaa !20 call void @llvm.dbg.value(metadata i32* %i, metadata !10, metadata !DIExpression(DW_OP_deref)), !dbg !24 call void @f2(i32* %i) #3, !dbg !24 ret i32 0, !dbg !25 } declare i32 @f3(i32) declare i32 @f1(...) declare void @f2(i32*) ; Function Attrs: nounwind readnone declare void @llvm.dbg.value(metadata, metadata, metadata) #2 attributes #0 = { nounwind ssp uwtable } attributes #2 = { nounwind readnone } attributes #3 = { nounwind } !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!11, !12} !llvm.ident = !{!13} !0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) !1 = !DIFile(filename: "dbg-value-const-byref.c", directory: "") !2 = !{} !4 = distinct !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !0, scopeLine: 5, file: !1, scope: !5, type: !6, retainedNodes: !9) !5 = !DIFile(filename: "dbg-value-const-byref.c", directory: "") !6 = !DISubroutineType(types: !7) !7 = !{!8} !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed) !9 = !{!10} !10 = !DILocalVariable(name: "i", line: 6, scope: !4, file: !5, type: !8) !11 = !{i32 2, !"Dwarf Version", i32 2} !12 = !{i32 1, !"Debug Info Version", i32 3} !13 = !{!"clang version 3.5.0 "} !14 = !{i32 3} !15 = !DILocation(line: 6, scope: !4) !16 = !DILocation(line: 7, scope: !4) !17 = !{i32 7} !18 = !DILocation(line: 8, scope: !4) !19 = !DILocation(line: 9, scope: !4) !20 = !{!21, !21, i64 0} !21 = !{!"int", !22, i64 0} !22 = !{!"omnipotent char", !23, i64 0} !23 = !{!"Simple C/C++ TBAA"} !24 = !DILocation(line: 10, scope: !4) !25 = !DILocation(line: 11, scope: !4) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-value-frame-index.ll000066400000000000000000000034501506646577400253170ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=x86_64-unknown-unknown -o - %t.ll | FileCheck %s ; RUN: llc -mtriple=x86_64-unknown-unknown -filetype=obj < %t.ll \ ; RUN: | llvm-dwarfdump -v - | FileCheck %s --check-prefix=DWARF target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" target triple = "spir64-unknown-unknown" define i1 @test() !dbg !4 { entry: %end = alloca i64, align 8 br label %while.cond while.cond: call void @llvm.dbg.value(metadata i64* %end, metadata !5, metadata !6), !dbg !7 %call = call i1 @fn(i64* %end, i64* %end, i64* null, i8* null, i64 0, i64* null, i32* null, i8* null), !dbg !7 br label %while.body while.body: br i1 0, label %while.end, label %while.cond while.end: ret i1 true } ; CHECK-LABEL: test ; CHECK: #DEBUG_VALUE: test:w <- [DW_OP_plus_uconst 8, DW_OP_deref] $rsp ; DWARF: DW_AT_location [DW_FORM_sec_offset] ( ; DWARF-NEXT: {{.*}}, {{.*}}: DW_OP_breg7 RSP+8) declare i1 @fn(i64*, i64*, i64*, i8*, i64, i64*, i32*, i8*) declare void @llvm.dbg.value(metadata, metadata, metadata) !llvm.dbg.cu = !{!0} !llvm.module.flags = !{!2,!3} !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0", emissionKind: FullDebug) !1 = !DIFile(filename: "test.c", directory: "/") !2 = !{i32 2, !"Dwarf Version", i32 4} !3 = !{i32 2, !"Debug Info Version", i32 3} !4 = distinct !DISubprogram(name: "test", type: !10, unit: !0) !5 = !DILocalVariable(name: "w", scope: !4, type: !9) !6 = !DIExpression(DW_OP_deref) !7 = !DILocation(line: 210, column: 12, scope: !4) !8 = !{!9} !9 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean) !10 = !DISubroutineType(types: !8) SPIRV-LLVM-Translator-15.0.17/test/DebugInfo/X86/dbg-value-isel.ll000066400000000000000000000111531506646577400240530ustar00rootroot00000000000000; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv -r %t.spv -o - | llvm-dis -o %t.ll ; RUN: llc -mtriple=%triple < %t.ll | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" target triple = "spir64-unknown-unknown" ; PR 9879 ; CHECK: #DEBUG_VALUE: tid <- %0 = type { i8*, i8*, i8*, i8*, i32 } @sgv = internal addrspace(2) constant [1 x i8] zeroinitializer @fgv = internal addrspace(2) constant [1 x i8] zeroinitializer define void @__OpenCL_nbt02_kernel(i32 addrspace(1)* %ip) nounwind !dbg !0 { entry: call void @llvm.dbg.value(metadata i32 addrspace(1)* %ip, metadata !8, metadata !DIExpression()), !dbg !9 %0 = call <4 x i32> @__amdil_get_local_id_int() nounwind %1 = extractelement <4 x i32> %0, i32 0 br label %2 ;