.clang-format

 Language: Cpp
 IndentWidth: 4
+PointerAlignment: Left
 BreakBeforeBraces: Custom
-BraceWrapping: { AfterFunction: true, AfterControlStatement: true }
+BraceWrapping: { AfterFunction: true, AfterControlStatement: false }
 IndentCaseLabels: false
 ReflowComments: false
 ColumnLimit: 120

BUILD.gn

@@ -152,6 +152,7 @@ source_set("glslang_sources") {
   if (is_win && !is_clang) {
     cflags = [
       "/wd4018", # signed/unsigned mismatch
+      "/wd4189", # local variable is initialized but not referenced
     ]
   }
 
@@ -169,7 +170,7 @@ source_set("glslang_default_resource_limits_sources") {
   public_configs = [ ":glslang_public" ]
 }
 
-source_set("glslang_validator") {
+executable("glslang_validator") {
   sources = [
     "StandAlone/DirStackFileIncluder.h",
     "StandAlone/StandAlone.cpp",

External/CMakeLists.txt

@@ -10,7 +10,8 @@ if(BUILD_TESTING)
         if(WIN32)
             set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
         endif(WIN32)
-        add_subdirectory(googletest)
+        # EXCLUDE_FROM_ALL keeps the install target from installing GTEST files.
+        add_subdirectory(googletest EXCLUDE_FROM_ALL)
         set(GTEST_TARGETS
             gtest
             gtest_main

SPIRV/GLSL.ext.NV.h

@@ -72,4 +72,7 @@ const char* const E_SPV_NV_ray_tracing = "SPV_NV_ray_tracing";
 //SPV_NV_shading_rate
 const char* const E_SPV_NV_shading_rate = "SPV_NV_shading_rate";
 
+//SPV_NV_cooperative_matrix
+const char* const E_SPV_NV_cooperative_matrix = "SPV_NV_cooperative_matrix";
+
 #endif  // #ifndef GLSLextNV_H

SPIRV/GlslangToSpv.cpp

@@ -49,9 +49,7 @@ namespace spv {
 #ifdef AMD_EXTENSIONS
     #include "GLSL.ext.AMD.h"
 #endif
-#ifdef NV_EXTENSIONS
     #include "GLSL.ext.NV.h"
-#endif
 }
 
 // Glslang includes
@@ -211,15 +209,6 @@ protected:
             builder.addExtension(ext);
     }
 
-    unsigned int getBufferReferenceAlignment(const glslang::TType &type) const {
-        if (type.getBasicType() == glslang::EbtReference) {
-            return type.getReferentType()->getQualifier().hasBufferReferenceAlign() ?
-                        (1u << type.getReferentType()->getQualifier().layoutBufferReferenceAlign) : 16u;
-        } else {
-            return 0;
-        }
-    }
-
     glslang::SpvOptions& options;
     spv::Function* shaderEntry;
     spv::Function* currentFunction;
@@ -567,14 +556,15 @@ spv::Builder::AccessChain::CoherentFlags TGlslangToSpvTraverser::TranslateCohere
     flags.workgroupcoherent = type.getQualifier().workgroupcoherent ||
                               type.getQualifier().storage == glslang::EvqShared;
     flags.subgroupcoherent = type.getQualifier().subgroupcoherent;
+    flags.volatil = type.getQualifier().volatil;
     // *coherent variables are implicitly nonprivate in GLSL
     flags.nonprivate = type.getQualifier().nonprivate ||
                        flags.subgroupcoherent ||
                        flags.workgroupcoherent ||
                        flags.queuefamilycoherent ||
                        flags.devicecoherent ||
-                       flags.coherent;
-    flags.volatil = type.getQualifier().volatil;
+                       flags.coherent ||
+                       flags.volatil;
     flags.isImage = type.getBasicType() == glslang::EbtSampler;
     return flags;
 }
@@ -582,7 +572,7 @@ spv::Builder::AccessChain::CoherentFlags TGlslangToSpvTraverser::TranslateCohere
 spv::Scope TGlslangToSpvTraverser::TranslateMemoryScope(const spv::Builder::AccessChain::CoherentFlags &coherentFlags)
 {
     spv::Scope scope;
-    if (coherentFlags.coherent) {
+    if (coherentFlags.volatil || coherentFlags.coherent) {
         // coherent defaults to Device scope in the old model, QueueFamilyKHR scope in the new model
         scope = glslangIntermediate->usingVulkanMemoryModel() ? spv::ScopeQueueFamilyKHR : spv::ScopeDevice;
     } else if (coherentFlags.devicecoherent) {
@@ -1330,6 +1320,10 @@ TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion, const gl
     }
     builder.setMemoryModel(addressingModel, memoryModel);
 
+    if (glslangIntermediate->usingVariablePointers()) {
+        builder.addCapability(spv::CapabilityVariablePointers);
+    }
+
     shaderEntry = builder.makeEntryPoint(glslangIntermediate->getEntryPointName().c_str());
     entryPoint = builder.addEntryPoint(executionModel, shaderEntry, glslangIntermediate->getEntryPointName().c_str());
 
@@ -1732,7 +1726,7 @@ bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::T
                 }
 
                 // normal case for indexing array or structure or block
-                builder.accessChainPush(builder.makeIntConstant(spvIndex), TranslateCoherent(node->getLeft()->getType()), getBufferReferenceAlignment(node->getLeft()->getType()));
+                builder.accessChainPush(builder.makeIntConstant(spvIndex), TranslateCoherent(node->getLeft()->getType()), node->getLeft()->getType().getBufferReferenceAlignment());
 
                 // Add capabilities here for accessing PointSize and clip/cull distance.
                 // We have deferred generation of associated capabilities until now.
@@ -1771,7 +1765,7 @@ bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::T
                                                 TranslateCoherent(node->getLeft()->getType()),
                                                 glslangIntermediate->getBaseAlignmentScalar(node->getLeft()->getType(), dummySize));
             } else
-                builder.accessChainPush(index, TranslateCoherent(node->getLeft()->getType()), getBufferReferenceAlignment(node->getLeft()->getType()));
+                builder.accessChainPush(index, TranslateCoherent(node->getLeft()->getType()), node->getLeft()->getType().getBufferReferenceAlignment());
         }
         return false;
     case glslang::EOpVectorSwizzle:
@@ -1870,16 +1864,31 @@ bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TI
         // So, this has to be block.lastMember.length().
         // SPV wants "block" and member number as the operands, go get them.
 
+        spv::Id length;
+        if (node->getOperand()->getType().isCoopMat()) {
+            spec_constant_op_mode_setter.turnOnSpecConstantOpMode();
+
+            spv::Id typeId = convertGlslangToSpvType(node->getOperand()->getType());
+            assert(builder.isCooperativeMatrixType(typeId));
+
+            length = builder.createCooperativeMatrixLength(typeId);
+        } else {
             glslang::TIntermTyped* block = node->getOperand()->getAsBinaryNode()->getLeft();
             block->traverse(this);
             unsigned int member = node->getOperand()->getAsBinaryNode()->getRight()->getAsConstantUnion()->getConstArray()[0].getUConst();
-        spv::Id length = builder.createArrayLength(builder.accessChainGetLValue(), member);
+            length = builder.createArrayLength(builder.accessChainGetLValue(), member);
+        }
 
         // GLSL semantics say the result of .length() is an int, while SPIR-V says
         // signedness must be 0. So, convert from SPIR-V unsigned back to GLSL's
         // AST expectation of a signed result.
-        if (glslangIntermediate->getSource() == glslang::EShSourceGlsl)
+        if (glslangIntermediate->getSource() == glslang::EShSourceGlsl) {
+            if (builder.isInSpecConstCodeGenMode()) {
+                length = builder.createBinOp(spv::OpIAdd, builder.makeIntType(32), length, builder.makeIntConstant(0));
+            } else {
                 length = builder.createUnaryOp(spv::OpBitcast, builder.makeIntType(32), length);
+            }
+        }
 
         builder.clearAccessChain();
         builder.setAccessChainRValue(length);
@@ -2222,6 +2231,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
     case glslang::EOpConstructStruct:
     case glslang::EOpConstructTextureSampler:
     case glslang::EOpConstructReference:
+    case glslang::EOpConstructCooperativeMatrix:
     {
         builder.setLine(node->getLoc().line, node->getLoc().getFilename());
         std::vector<spv::Id> arguments;
@@ -2229,7 +2239,9 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
         spv::Id constructed;
         if (node->getOp() == glslang::EOpConstructTextureSampler)
             constructed = builder.createOp(spv::OpSampledImage, resultType(), arguments);
-        else if (node->getOp() == glslang::EOpConstructStruct || node->getType().isArray()) {
+        else if (node->getOp() == glslang::EOpConstructStruct ||
+                 node->getOp() == glslang::EOpConstructCooperativeMatrix ||
+                 node->getType().isArray()) {
             std::vector<spv::Id> constituents;
             for (int c = 0; c < (int)arguments.size(); ++c)
                 constituents.push_back(arguments[c]);
@@ -2347,6 +2359,10 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
         noReturnValue = true;
         break;
 #endif
+    case glslang::EOpCooperativeMatrixLoad:
+    case glslang::EOpCooperativeMatrixStore:
+        noReturnValue = true;
+        break;
 
     default:
         break;
@@ -2389,6 +2405,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
     //
     glslang::TIntermSequence& glslangOperands = node->getSequence();
     std::vector<spv::Id> operands;
+    std::vector<spv::IdImmediate> memoryAccessOperands;
     for (int arg = 0; arg < (int)glslangOperands.size(); ++arg) {
         // special case l-value operands; there are just a few
         bool lvalue = false;
@@ -2445,6 +2462,14 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
             if (arg >= 2)
                 lvalue = true;
             break;
+        case glslang::EOpCooperativeMatrixLoad:
+            if (arg == 0 || arg == 1)
+                lvalue = true;
+            break;
+        case glslang::EOpCooperativeMatrixStore:
+            if (arg == 1)
+                lvalue = true;
+            break;
         default:
             break;
         }
@@ -2453,6 +2478,50 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
             glslangOperands[0]->getAsBinaryNode()->getLeft()->traverse(this);
         else
             glslangOperands[arg]->traverse(this);
+
+        if (node->getOp() == glslang::EOpCooperativeMatrixLoad ||
+            node->getOp() == glslang::EOpCooperativeMatrixStore) {
+
+            if (arg == 1) {
+                // fold "element" parameter into the access chain
+                spv::Builder::AccessChain save = builder.getAccessChain();
+                builder.clearAccessChain();
+                glslangOperands[2]->traverse(this);
+
+                spv::Id elementId = accessChainLoad(glslangOperands[2]->getAsTyped()->getType());
+
+                builder.setAccessChain(save);
+
+                // Point to the first element of the array.
+                builder.accessChainPush(elementId, TranslateCoherent(glslangOperands[arg]->getAsTyped()->getType()),
+                                                   glslangOperands[arg]->getAsTyped()->getType().getBufferReferenceAlignment());
+
+                spv::Builder::AccessChain::CoherentFlags coherentFlags = builder.getAccessChain().coherentFlags;
+                unsigned int alignment = builder.getAccessChain().alignment;
+
+                int memoryAccess = TranslateMemoryAccess(coherentFlags);
+                if (node->getOp() == glslang::EOpCooperativeMatrixLoad)
+                    memoryAccess &= ~spv::MemoryAccessMakePointerAvailableKHRMask;
+                if (node->getOp() == glslang::EOpCooperativeMatrixStore)
+                    memoryAccess &= ~spv::MemoryAccessMakePointerVisibleKHRMask;
+                if (builder.getStorageClass(builder.getAccessChain().base) == spv::StorageClassPhysicalStorageBufferEXT) {
+                    memoryAccess = (spv::MemoryAccessMask)(memoryAccess | spv::MemoryAccessAlignedMask);
+                }
+
+                memoryAccessOperands.push_back(spv::IdImmediate(false, memoryAccess));
+
+                if (memoryAccess & spv::MemoryAccessAlignedMask) {
+                    memoryAccessOperands.push_back(spv::IdImmediate(false, alignment));
+                }
+
+                if (memoryAccess & (spv::MemoryAccessMakePointerAvailableKHRMask | spv::MemoryAccessMakePointerVisibleKHRMask)) {
+                    memoryAccessOperands.push_back(spv::IdImmediate(true, builder.makeUintConstant(TranslateMemoryScope(coherentFlags))));
+                }
+            } else if (arg == 2) {
+                continue;
+            }
+        }
+
         if (lvalue)
             operands.push_back(builder.accessChainGetLValue());
         else {
@@ -2462,7 +2531,33 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt
     }
 
     builder.setLine(node->getLoc().line, node->getLoc().getFilename());
-    if (atomic) {
+    if (node->getOp() == glslang::EOpCooperativeMatrixLoad) {
+        std::vector<spv::IdImmediate> idImmOps;
+
+        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
+        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
+        idImmOps.push_back(spv::IdImmediate(true, operands[3])); // colMajor
+        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
+        // get the pointee type
+        spv::Id typeId = builder.getContainedTypeId(builder.getTypeId(operands[0]));
+        assert(builder.isCooperativeMatrixType(typeId));
+        // do the op
+        spv::Id result = builder.createOp(spv::OpCooperativeMatrixLoadNV, typeId, idImmOps);
+        // store the result to the pointer (out param 'm')
+        builder.createStore(result, operands[0]);
+        result = 0;
+    } else if (node->getOp() == glslang::EOpCooperativeMatrixStore) {
+        std::vector<spv::IdImmediate> idImmOps;
+
+        idImmOps.push_back(spv::IdImmediate(true, operands[1])); // buf
+        idImmOps.push_back(spv::IdImmediate(true, operands[0])); // object
+        idImmOps.push_back(spv::IdImmediate(true, operands[2])); // stride
+        idImmOps.push_back(spv::IdImmediate(true, operands[3])); // colMajor
+        idImmOps.insert(idImmOps.end(), memoryAccessOperands.begin(), memoryAccessOperands.end());
+
+        builder.createNoResultOp(spv::OpCooperativeMatrixStoreNV, idImmOps);
+        result = 0;
+    } else if (atomic) {
         // Handle all atomics
         result = createAtomicOperation(node->getOp(), precision, resultType(), operands, node->getBasicType());
     } else {
@@ -3090,6 +3185,19 @@ spv::Id TGlslangToSpvTraverser::convertGlslangToSpvType(const glslang::TType& ty
             spvType = builder.makeVectorType(spvType, type.getVectorSize());
     }
 
+    if (type.isCoopMat()) {
+        builder.addCapability(spv::CapabilityCooperativeMatrixNV);
+        builder.addExtension(spv::E_SPV_NV_cooperative_matrix);
+        if (type.getBasicType() == glslang::EbtFloat16)
+            builder.addCapability(spv::CapabilityFloat16);
+
+        spv::Id scope = makeArraySizeId(*type.getTypeParameters(), 1);
+        spv::Id rows = makeArraySizeId(*type.getTypeParameters(), 2);
+        spv::Id cols = makeArraySizeId(*type.getTypeParameters(), 3);
+
+        spvType = builder.makeCooperativeMatrixType(spvType, scope, rows, cols);
+    }
+
     if (type.isArray()) {
         int stride = 0;  // keep this 0 unless doing an explicit layout; 0 will mean no decoration, no stride
 
@@ -3401,7 +3509,7 @@ spv::Id TGlslangToSpvTraverser::accessChainLoad(const glslang::TType& type)
     coherentFlags |= TranslateCoherent(type);
 
     unsigned int alignment = builder.getAccessChain().alignment;
-    alignment |= getBufferReferenceAlignment(type);
+    alignment |= type.getBufferReferenceAlignment();
 
     spv::Id loadedId = builder.accessChainLoad(TranslatePrecisionDecoration(type),
                                                TranslateNonUniformDecoration(type.getQualifier()),
@@ -3468,7 +3576,7 @@ void TGlslangToSpvTraverser::accessChainStore(const glslang::TType& type, spv::I
     coherentFlags |= TranslateCoherent(type);
 
     unsigned int alignment = builder.getAccessChain().alignment;
-    alignment |= getBufferReferenceAlignment(type);
+    alignment |= type.getBufferReferenceAlignment();
 
     builder.accessChainStore(rvalue,
                              spv::MemoryAccessMask(TranslateMemoryAccess(coherentFlags) & ~spv::MemoryAccessMakePointerVisibleKHRMask),
@@ -3518,7 +3626,7 @@ void TGlslangToSpvTraverser::multiTypeStore(const glslang::TType& type, spv::Id
             // set up the target storage
             builder.clearAccessChain();
             builder.setAccessChainLValue(lValue);
-            builder.accessChainPush(builder.makeIntConstant(index), TranslateCoherent(type), getBufferReferenceAlignment(type));
+            builder.accessChainPush(builder.makeIntConstant(index), TranslateCoherent(type), type.getBufferReferenceAlignment());
 
             // store the member
             multiTypeStore(glslangElementType, elementRValue);
@@ -3538,7 +3646,7 @@ void TGlslangToSpvTraverser::multiTypeStore(const glslang::TType& type, spv::Id
             // set up the target storage
             builder.clearAccessChain();
             builder.setAccessChainLValue(lValue);
-            builder.accessChainPush(builder.makeIntConstant(m), TranslateCoherent(type), getBufferReferenceAlignment(type));
+            builder.accessChainPush(builder.makeIntConstant(m), TranslateCoherent(type), type.getBufferReferenceAlignment());
 
             // store the member
             multiTypeStore(glslangMemberType, memberRValue);
@@ -4847,7 +4955,8 @@ spv::Id TGlslangToSpvTraverser::createBinaryOperation(glslang::TOperator op, OpD
     // handle mapped binary operations (should be non-comparison)
     if (binOp != spv::OpNop) {
         assert(comparison == false);
-        if (builder.isMatrix(left) || builder.isMatrix(right))
+        if (builder.isMatrix(left) || builder.isMatrix(right) ||
+            builder.isCooperativeMatrix(left) || builder.isCooperativeMatrix(right))
             return createBinaryMatrixOperation(binOp, decorations, typeId, left, right);
 
         // No matrix involved; make both operands be the same number of components, if needed
@@ -4968,7 +5077,7 @@ spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecora
             firstClass = false;
         break;
     case spv::OpMatrixTimesScalar:
-        if (builder.isMatrix(right))
+        if (builder.isMatrix(right) || builder.isCooperativeMatrix(right))
             std::swap(left, right);
         assert(builder.isScalar(right));
         break;
@@ -4989,6 +5098,9 @@ spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecora
         break;
     }
 
+    if (builder.isCooperativeMatrix(left) || builder.isCooperativeMatrix(right))
+        firstClass = true;
+
     if (firstClass) {
         spv::Id result = builder.createBinOp(op, typeId, left, right);
         builder.addDecoration(result, decorations.noContraction);
@@ -5431,6 +5543,11 @@ spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDe
     case glslang::EOpConstructReference:
         unaryOp = spv::OpBitcast;
         break;
+
+    case glslang::EOpCopyObject:
+        unaryOp = spv::OpCopyObject;
+        break;
+
     default:
         return 0;
     }
@@ -7030,6 +7147,10 @@ spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv::
         builder.createNoResultOp(spv::OpWritePackedPrimitiveIndices4x8NV, operands);
         return 0;
 #endif
+    case glslang::EOpCooperativeMatrixMulAdd:
+        opCode = spv::OpCooperativeMatrixMulAddNV;
+        break;
+
     default:
         return 0;
     }
@@ -7486,6 +7607,9 @@ spv::Id TGlslangToSpvTraverser::createSpvConstantFromConstUnionArray(const glsla
         glslang::TType vectorType(glslangType, 0);
         for (int col = 0; col < glslangType.getMatrixCols(); ++col)
             spvConsts.push_back(createSpvConstantFromConstUnionArray(vectorType, consts, nextConst, false));
+    } else if (glslangType.isCoopMat()) {
+        glslang::TType componentType(glslangType.getBasicType());
+        spvConsts.push_back(createSpvConstantFromConstUnionArray(componentType, consts, nextConst, false));
     } else if (glslangType.isStruct()) {
         glslang::TVector<glslang::TTypeLoc>::const_iterator iter;
         for (iter = glslangType.getStruct()->begin(); iter != glslangType.getStruct()->end(); ++iter)
@@ -7577,6 +7701,10 @@ spv::Id TGlslangToSpvTraverser::createSpvConstantFromConstUnionArray(const glsla
         case glslang::EbtBool:
             scalar = builder.makeBoolConstant(zero ? false : consts[nextConst].getBConst(), specConstant);
             break;
+        case glslang::EbtReference:
+            scalar = builder.makeUint64Constant(zero ? 0 : consts[nextConst].getU64Const(), specConstant);
+            scalar = builder.createUnaryOp(spv::OpBitcast, typeId, scalar);
+            break;
         default:
             assert(0);
             break;

SPIRV/SPVRemapper.h

@@ -45,7 +45,7 @@ namespace spv {
 
 // MSVC defines __cplusplus as an older value, even when it supports almost all of 11.
 // We handle that here by making our own symbol.
-#if __cplusplus >= 201103L || _MSC_VER >= 1700
+#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1700)
 #   define use_cpp11 1
 #endif
 

SPIRV/SpvBuilder.cpp

@@ -388,6 +388,33 @@ Id Builder::makeMatrixType(Id component, int cols, int rows)
     return type->getResultId();
 }
 
+Id Builder::makeCooperativeMatrixType(Id component, Id scope, Id rows, Id cols)
+{
+    // try to find it
+    Instruction* type;
+    for (int t = 0; t < (int)groupedTypes[OpTypeCooperativeMatrixNV].size(); ++t) {
+        type = groupedTypes[OpTypeCooperativeMatrixNV][t];
+        if (type->getIdOperand(0) == component &&
+            type->getIdOperand(1) == scope &&
+            type->getIdOperand(2) == rows &&
+            type->getIdOperand(3) == cols)
+            return type->getResultId();
+    }
+
+    // not found, make it
+    type = new Instruction(getUniqueId(), NoType, OpTypeCooperativeMatrixNV);
+    type->addIdOperand(component);
+    type->addIdOperand(scope);
+    type->addIdOperand(rows);
+    type->addIdOperand(cols);
+    groupedTypes[OpTypeCooperativeMatrixNV].push_back(type);
+    constantsTypesGlobals.push_back(std::unique_ptr<Instruction>(type));
+    module.mapInstruction(type);
+
+    return type->getResultId();
+}
+
+
 // TODO: performance: track arrays per stride
 // If a stride is supplied (non-zero) make an array.
 // If no stride (0), reuse previous array types.
@@ -623,6 +650,9 @@ int Builder::getNumTypeConstituents(Id typeId) const
     }
     case OpTypeStruct:
         return instr->getNumOperands();
+    case OpTypeCooperativeMatrixNV:
+        // has only one constituent when used with OpCompositeConstruct.
+        return 1;
     default:
         assert(0);
         return 1;
@@ -669,6 +699,7 @@ Id Builder::getContainedTypeId(Id typeId, int member) const
     case OpTypeMatrix:
     case OpTypeArray:
     case OpTypeRuntimeArray:
+    case OpTypeCooperativeMatrixNV:
         return instr->getIdOperand(0);
     case OpTypePointer:
         return instr->getIdOperand(1);
@@ -981,15 +1012,14 @@ Id Builder::makeFpConstant(Id type, double d, bool specConstant)
         return NoResult;
 }
 
-Id Builder::findCompositeConstant(Op typeClass, const std::vector<Id>& comps)
+Id Builder::findCompositeConstant(Op typeClass, Id typeId, const std::vector<Id>& comps)
 {
     Instruction* constant = 0;
     bool found = false;
     for (int i = 0; i < (int)groupedConstants[typeClass].size(); ++i) {
         constant = groupedConstants[typeClass][i];
 
-        // same shape?
-        if (constant->getNumOperands() != (int)comps.size())
+        if (constant->getTypeId() != typeId)
             continue;
 
         // same contents?
@@ -1044,8 +1074,9 @@ Id Builder::makeCompositeConstant(Id typeId, const std::vector<Id>& members, boo
     case OpTypeVector:
     case OpTypeArray:
     case OpTypeMatrix:
+    case OpTypeCooperativeMatrixNV:
         if (! specConstant) {
-            Id existing = findCompositeConstant(typeClass, members);
+            Id existing = findCompositeConstant(typeClass, typeId, members);
             if (existing)
                 return existing;
         }
@@ -1408,6 +1439,23 @@ Id Builder::createArrayLength(Id base, unsigned int member)
     return length->getResultId();
 }
 
+Id Builder::createCooperativeMatrixLength(Id type)
+{
+    spv::Id intType = makeUintType(32);
+
+    // Generate code for spec constants if in spec constant operation
+    // generation mode.
+    if (generatingOpCodeForSpecConst) {
+        return createSpecConstantOp(OpCooperativeMatrixLengthNV, intType, std::vector<Id>(1, type), std::vector<Id>());
+    }
+
+    Instruction* length = new Instruction(getUniqueId(), intType, OpCooperativeMatrixLengthNV);
+    length->addIdOperand(type);
+    buildPoint->addInstruction(std::unique_ptr<Instruction>(length));
+
+    return length->getResultId();
+}
+
 Id Builder::createCompositeExtract(Id composite, Id typeId, unsigned index)
 {
     // Generate code for spec constants if in spec constant operation
@@ -2598,9 +2646,9 @@ Id Builder::accessChainLoad(Decoration precision, Decoration nonUniform, Id resu
                 }
             }
 
-            if (constant)
+            if (constant) {
                 id = createCompositeExtract(accessChain.base, swizzleBase, indexes);
-            else {
+            } else {
                 // make a new function variable for this r-value
                 Id lValue = createVariable(StorageClassFunction, getTypeId(accessChain.base), "indexable");
 

SPIRV/SpvBuilder.h

@@ -155,6 +155,7 @@ public:
     Id makeImageType(Id sampledType, Dim, bool depth, bool arrayed, bool ms, unsigned sampled, ImageFormat format);
     Id makeSamplerType();
     Id makeSampledImageType(Id imageType);
+    Id makeCooperativeMatrixType(Id component, Id scope, Id rows, Id cols);
 
     // accelerationStructureNV type
     Id makeAccelerationStructureNVType();
@@ -178,6 +179,7 @@ public:
     bool isScalar(Id resultId)       const { return isScalarType(getTypeId(resultId)); }
     bool isVector(Id resultId)       const { return isVectorType(getTypeId(resultId)); }
     bool isMatrix(Id resultId)       const { return isMatrixType(getTypeId(resultId)); }
+    bool isCooperativeMatrix(Id resultId)const { return isCooperativeMatrixType(getTypeId(resultId)); }
     bool isAggregate(Id resultId)    const { return isAggregateType(getTypeId(resultId)); }
     bool isSampledImage(Id resultId) const { return isSampledImageType(getTypeId(resultId)); }
 
@@ -191,7 +193,8 @@ public:
     bool isMatrixType(Id typeId)       const { return getTypeClass(typeId) == OpTypeMatrix; }
     bool isStructType(Id typeId)       const { return getTypeClass(typeId) == OpTypeStruct; }
     bool isArrayType(Id typeId)        const { return getTypeClass(typeId) == OpTypeArray; }
-    bool isAggregateType(Id typeId)    const { return isArrayType(typeId) || isStructType(typeId); }
+    bool isCooperativeMatrixType(Id typeId)const { return getTypeClass(typeId) == OpTypeCooperativeMatrixNV; }
+    bool isAggregateType(Id typeId)    const { return isArrayType(typeId) || isStructType(typeId) || isCooperativeMatrixType(typeId); }
     bool isImageType(Id typeId)        const { return getTypeClass(typeId) == OpTypeImage; }
     bool isSamplerType(Id typeId)      const { return getTypeClass(typeId) == OpTypeSampler; }
     bool isSampledImageType(Id typeId) const { return getTypeClass(typeId) == OpTypeSampledImage; }
@@ -314,6 +317,9 @@ public:
     // Create an OpArrayLength instruction
     Id createArrayLength(Id base, unsigned int member);
 
+    // Create an OpCooperativeMatrixLengthNV instruction
+    Id createCooperativeMatrixLength(Id type);
+
     // Create an OpCompositeExtract instruction
     Id createCompositeExtract(Id composite, Id typeId, unsigned index);
     Id createCompositeExtract(Id composite, Id typeId, const std::vector<unsigned>& indexes);
@@ -670,7 +676,7 @@ public:
     Id makeInt64Constant(Id typeId, unsigned long long value, bool specConstant);
     Id findScalarConstant(Op typeClass, Op opcode, Id typeId, unsigned value);
     Id findScalarConstant(Op typeClass, Op opcode, Id typeId, unsigned v1, unsigned v2);
-    Id findCompositeConstant(Op typeClass, const std::vector<Id>& comps);
+    Id findCompositeConstant(Op typeClass, Id typeId, const std::vector<Id>& comps);
     Id findStructConstant(Id typeId, const std::vector<Id>& comps);
     Id collapseAccessChain();
     void remapDynamicSwizzle();

SPIRV/SpvPostProcess.cpp

@@ -118,9 +118,46 @@ void Builder::postProcessType(const Instruction& inst, Id typeId)
     case OpAccessChain:
     case OpPtrAccessChain:
     case OpCopyObject:
+        break;
     case OpFConvert:
     case OpSConvert:
     case OpUConvert:
+        // Look for any 8/16-bit storage capabilities. If there are none, assume that
+        // the convert instruction requires the Float16/Int8/16 capability.
+        if (containsType(typeId, OpTypeFloat, 16) || containsType(typeId, OpTypeInt, 16)) {
+            bool foundStorage = false;
+            for (auto it = capabilities.begin(); it != capabilities.end(); ++it) {
+                spv::Capability cap = *it;
+                if (cap == spv::CapabilityStorageInputOutput16 ||
+                    cap == spv::CapabilityStoragePushConstant16 ||
+                    cap == spv::CapabilityStorageUniformBufferBlock16 ||
+                    cap == spv::CapabilityStorageUniform16) {
+                    foundStorage = true;
+                    break;
+                }
+            }
+            if (!foundStorage) {
+                if (containsType(typeId, OpTypeFloat, 16))
+                    addCapability(CapabilityFloat16);
+                if (containsType(typeId, OpTypeInt, 16))
+                    addCapability(CapabilityInt16);
+            }
+        }
+        if (containsType(typeId, OpTypeInt, 8)) {
+            bool foundStorage = false;
+            for (auto it = capabilities.begin(); it != capabilities.end(); ++it) {
+                spv::Capability cap = *it;
+                if (cap == spv::CapabilityStoragePushConstant8 ||
+                    cap == spv::CapabilityUniformAndStorageBuffer8BitAccess ||
+                    cap == spv::CapabilityStorageBuffer8BitAccess) {
+                    foundStorage = true;
+                    break;
+                }
+            }
+            if (!foundStorage) {
+                addCapability(CapabilityInt8);
+            }
+        }
         break;
     case OpExtInst:
 #if AMD_EXTENSIONS
@@ -258,10 +295,9 @@ void Builder::postProcess(Instruction& inst)
                 assert(inst.getNumOperands() >= 3);
                 unsigned int memoryAccess = inst.getImmediateOperand((inst.getOpCode() == OpStore) ? 2 : 1);
                 assert(memoryAccess & MemoryAccessAlignedMask);
+                static_cast<void>(memoryAccess);
                 // Compute the index of the alignment operand.
                 int alignmentIdx = 2;
-                if (memoryAccess & MemoryAccessVolatileMask)
-                    alignmentIdx++;
                 if (inst.getOpCode() == OpStore)
                     alignmentIdx++;
                 // Merge new and old (mis)alignment
@@ -328,6 +364,24 @@ void Builder::postProcess()
 
     // Add per-instruction capabilities, extensions, etc.,
 
+    // Look for any 8/16 bit type in physical storage buffer class, and set the
+    // appropriate capability. This happens in createSpvVariable for other storage
+    // classes, but there isn't always a variable for physical storage buffer.
+    for (int t = 0; t < (int)groupedTypes[OpTypePointer].size(); ++t) {
+        Instruction* type = groupedTypes[OpTypePointer][t];
+        if (type->getImmediateOperand(0) == (unsigned)StorageClassPhysicalStorageBufferEXT) {
+            if (containsType(type->getIdOperand(1), OpTypeInt, 8)) {
+                addExtension(spv::E_SPV_KHR_8bit_storage);
+                addCapability(spv::CapabilityStorageBuffer8BitAccess);
+            }
+            if (containsType(type->getIdOperand(1), OpTypeInt, 16) ||
+                containsType(type->getIdOperand(1), OpTypeFloat, 16)) {
+                addExtension(spv::E_SPV_KHR_16bit_storage);
+                addCapability(spv::CapabilityStorageBuffer16BitAccess);
+            }
+        }
+    }
+
     // process all reachable instructions...
     for (auto bi = reachableBlocks.cbegin(); bi != reachableBlocks.cend(); ++bi) {
         const Block* block = *bi;
@@ -367,24 +421,6 @@ void Builder::postProcess()
             }
         }
     }
-
-    // Look for any 8/16 bit type in physical storage buffer class, and set the
-    // appropriate capability. This happens in createSpvVariable for other storage
-    // classes, but there isn't always a variable for physical storage buffer.
-    for (int t = 0; t < (int)groupedTypes[OpTypePointer].size(); ++t) {
-        Instruction* type = groupedTypes[OpTypePointer][t];
-        if (type->getImmediateOperand(0) == (unsigned)StorageClassPhysicalStorageBufferEXT) {
-            if (containsType(type->getIdOperand(1), OpTypeInt, 8)) {
-                addExtension(spv::E_SPV_KHR_8bit_storage);
-                addCapability(spv::CapabilityStorageBuffer8BitAccess);
-            }
-            if (containsType(type->getIdOperand(1), OpTypeInt, 16) ||
-                containsType(type->getIdOperand(1), OpTypeFloat, 16)) {
-                addExtension(spv::E_SPV_KHR_16bit_storage);
-                addCapability(spv::CapabilityStorageBuffer16BitAccess);
-            }
-        }
-    }
 }
 
 }; // end spv namespace

SPIRV/SpvTools.cpp

@@ -191,7 +191,9 @@ void SpirvToolsLegalize(const glslang::TIntermediate&, std::vector<unsigned int>
         optimizer.RegisterPass(spvtools::CreateRedundantLineInfoElimPass());
     }
 
-    optimizer.Run(spirv.data(), spirv.size(), &spirv);
+    spvtools::OptimizerOptions spvOptOptions;
+    spvOptOptions.set_run_validator(false); // The validator may run as a seperate step later on
+    optimizer.Run(spirv.data(), spirv.size(), &spirv, spvOptOptions);
 }
 
 }; // end namespace glslang

SPIRV/SpvTools.h

@@ -75,6 +75,6 @@ void SpirvToolsLegalize(const glslang::TIntermediate& intermediate, std::vector<
 
 #endif
 
-}; // end namespace glslang
+} // end namespace glslang
 
 #endif // GLSLANG_SPV_TOOLS_H

SPIRV/disassemble.h

@@ -48,6 +48,6 @@ namespace spv {
     // disassemble with glslang custom disassembler
     void Disassemble(std::ostream& out, const std::vector<unsigned int>&);
 
-};  // end namespace spv
+}  // end namespace spv
 
 #endif // disassembler_H

SPIRV/doc.cpp

@@ -930,6 +930,10 @@ const char* CapabilityString(int info)
 
     case CapabilityPhysicalStorageBufferAddressesEXT:   return "CapabilityPhysicalStorageBufferAddressesEXT";
 
+    case CapabilityVariablePointers:                    return "CapabilityVariablePointers";
+
+    case CapabilityCooperativeMatrixNV:     return "CapabilityCooperativeMatrixNV";
+
     default: return "Bad";
     }
 }
@@ -1333,6 +1337,12 @@ const char* OpcodeString(int op)
     case OpWritePackedPrimitiveIndices4x8NV: return "OpWritePackedPrimitiveIndices4x8NV";
 #endif
 
+    case OpTypeCooperativeMatrixNV:         return "OpTypeCooperativeMatrixNV";
+    case OpCooperativeMatrixLoadNV:         return "OpCooperativeMatrixLoadNV";
+    case OpCooperativeMatrixStoreNV:        return "OpCooperativeMatrixStoreNV";
+    case OpCooperativeMatrixMulAddNV:       return "OpCooperativeMatrixMulAddNV";
+    case OpCooperativeMatrixLengthNV:       return "OpCooperativeMatrixLengthNV";
+
     default:
         return "Bad";
     }
@@ -1444,6 +1454,8 @@ void Parameterize()
     InstructionDesc[OpGroupWaitEvents].setResultAndType(false, false);
     InstructionDesc[OpAtomicFlagClear].setResultAndType(false, false);
     InstructionDesc[OpModuleProcessed].setResultAndType(false, false);
+    InstructionDesc[OpTypeCooperativeMatrixNV].setResultAndType(true, false);
+    InstructionDesc[OpCooperativeMatrixStoreNV].setResultAndType(false, false);
 
     // Specific additional context-dependent operands
 
@@ -2714,6 +2726,32 @@ void Parameterize()
     InstructionDesc[OpWritePackedPrimitiveIndices4x8NV].operands.push(OperandId, "'Index Offset'");
     InstructionDesc[OpWritePackedPrimitiveIndices4x8NV].operands.push(OperandId, "'Packed Indices'");
 #endif
+
+    InstructionDesc[OpTypeCooperativeMatrixNV].operands.push(OperandId, "'Component Type'");
+    InstructionDesc[OpTypeCooperativeMatrixNV].operands.push(OperandId, "'Scope'");
+    InstructionDesc[OpTypeCooperativeMatrixNV].operands.push(OperandId, "'Rows'");
+    InstructionDesc[OpTypeCooperativeMatrixNV].operands.push(OperandId, "'Columns'");
+
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandId, "'Pointer'");
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandId, "'Stride'");
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandId, "'Column Major'");
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandMemoryAccess, "'Memory Access'");
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandLiteralNumber, "", true);
+    InstructionDesc[OpCooperativeMatrixLoadNV].operands.push(OperandId, "", true);
+
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandId, "'Pointer'");
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandId, "'Object'");
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandId, "'Stride'");
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandId, "'Column Major'");
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandMemoryAccess, "'Memory Access'");
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandLiteralNumber, "", true);
+    InstructionDesc[OpCooperativeMatrixStoreNV].operands.push(OperandId, "", true);
+
+    InstructionDesc[OpCooperativeMatrixMulAddNV].operands.push(OperandId, "'A'");
+    InstructionDesc[OpCooperativeMatrixMulAddNV].operands.push(OperandId, "'B'");
+    InstructionDesc[OpCooperativeMatrixMulAddNV].operands.push(OperandId, "'C'");
+
+    InstructionDesc[OpCooperativeMatrixLengthNV].operands.push(OperandId, "'Type'");
 }
 
 }; // end spv namespace

SPIRV/doc.h

@@ -255,4 +255,4 @@ const char* AccessQualifierString(int attr);
 
 void PrintOperands(const OperandParameters& operands, int reservedOperands);
 
-};  // end namespace spv
+}  // end namespace spv

SPIRV/spvIR.h

@@ -83,6 +83,7 @@ const MemorySemanticsMask MemorySemanticsAllMemory =
 struct IdImmediate {
     bool isId;      // true if word is an Id, false if word is an immediate
     unsigned word;
+    IdImmediate(bool i, unsigned w) : isId(i), word(w) {}
 };
 
 //

StandAlone/StandAlone.cpp

@@ -152,6 +152,7 @@ void ProcessConfigFile()
     }
 }
 
+int ReflectOptions = EShReflectionDefault;
 int Options = 0;
 const char* ExecutableName = nullptr;
 const char* binaryFileName = nullptr;
@@ -161,6 +162,7 @@ const char* shaderStageName = nullptr;
 const char* variableName = nullptr;
 bool HlslEnable16BitTypes = false;
 bool HlslDX9compatible = false;
+bool DumpBuiltinSymbols = false;
 std::vector<std::string> IncludeDirectoryList;
 
 // Source environment
@@ -493,6 +495,8 @@ void ProcessArguments(std::vector<std::unique_ptr<glslang::TWorkItem>>& workItem
                                 Error("--client expects vulkan100 or opengl100");
                         }
                         bumpArg();
+                    } else if (lowerword == "dump-builtin-symbols") {
+                        DumpBuiltinSymbols = true;
                     } else if (lowerword == "entry-point") {
                         entryPointName = argv[1];
                         if (argc <= 1)
@@ -523,6 +527,18 @@ void ProcessArguments(std::vector<std::unique_ptr<glslang::TWorkItem>>& workItem
                         Options |= EOptionNoStorageFormat;
                     } else if (lowerword == "relaxed-errors") {
                         Options |= EOptionRelaxedErrors;
+                    } else if (lowerword == "reflect-strict-array-suffix") {
+                        ReflectOptions |= EShReflectionStrictArraySuffix;
+                    } else if (lowerword == "reflect-basic-array-suffix") {
+                        ReflectOptions |= EShReflectionBasicArraySuffix;
+                    } else if (lowerword == "reflect-intermediate-io") {
+                        ReflectOptions |= EShReflectionIntermediateIO;
+                    } else if (lowerword == "reflect-separate-buffers") {
+                        ReflectOptions |= EShReflectionSeparateBuffers;
+                    } else if (lowerword == "reflect-all-block-variables") {
+                        ReflectOptions |= EShReflectionAllBlockVariables;
+                    } else if (lowerword == "reflect-unwrap-io-blocks") {
+                        ReflectOptions |= EShReflectionUnwrapIOBlocks;
                     } else if (lowerword == "resource-set-bindings" ||  // synonyms
                                lowerword == "resource-set-binding"  ||
                                lowerword == "rsb") {
@@ -820,6 +836,8 @@ void SetMessageOptions(EShMessages& messages)
         messages = (EShMessages)(messages | EShMsgHlslLegalization);
     if (HlslDX9compatible)
         messages = (EShMessages)(messages | EShMsgHlslDX9Compatible);
+    if (DumpBuiltinSymbols)
+        messages = (EShMessages)(messages | EShMsgBuiltinSymbolTable);
 }
 
 //
@@ -1051,7 +1069,7 @@ void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
 
     // Reflect
     if (Options & EOptionDumpReflection) {
-        program.buildReflection();
+        program.buildReflection(ReflectOptions);
         program.dumpReflection();
     }
 
@@ -1507,6 +1525,7 @@ void usage()
            "  --auto-map-locations | --aml      automatically locate input/output lacking\n"
            "                                    'location' (fragile, not cross stage)\n"
            "  --client {vulkan<ver>|opengl<ver>} see -V and -G\n"
+           "  --dump-builtin-symbols            prints builtin symbol table prior each compile\n"
            "  -dumpfullversion | -dumpversion   print bare major.minor.patchlevel\n"
            "  --flatten-uniform-arrays | --fua  flatten uniform texture/sampler arrays to\n"
            "                                    scalars\n"
@@ -1518,6 +1537,17 @@ void usage()
            "  --invert-y | --iy                 invert position.Y output in vertex shader\n"
            "  --keep-uncalled | --ku            don't eliminate uncalled functions\n"
            "  --no-storage-format | --nsf       use Unknown image format\n"
+           "  --reflect-strict-array-suffix     use strict array suffix rules when\n"
+           "                                    reflecting\n"
+           "  --reflect-basic-array-suffix      arrays of basic types will have trailing [0]\n"
+           "  --reflect-intermediate-io         reflection includes inputs/outputs of linked\n"
+           "                                    shaders rather than just vertex/fragment\n"
+           "  --reflect-separate-buffers        reflect buffer variables and blocks\n"
+           "                                    separately to uniforms\n"
+           "  --reflect-all-block-variables     reflect all variables in blocks, whether\n"
+           "                                    inactive or active\n"
+           "  --reflect-unwrap-io-blocks        unwrap input/output blocks the same as\n"
+           "                                    uniform blocks\n"
            "  --resource-set-binding [stage] name set binding\n"
            "                                    set descriptor set and binding for\n"
            "                                    individual resources\n"

Test/310.comp

@@ -106,9 +106,9 @@ layout(r32i) coherent restrict readonly uniform iimage2D qualim2;
 
 void passrc()
 {
-    passr(qualim1);
-    passr(qualim2);   // ERROR, drops restrict
-    passr(iimg2D);
+    passr(qualim1);   // ERROR, changing formats
+    passr(qualim2);   // ERROR, drops restrict, ERROR, changing formats
+    passr(iimg2D);    // ERROR, changing formats
 }
 
 highp layout(rg8i)     uniform readonly uimage2D i1bad; // ERROR, type mismatch

Test/310.tesc

@@ -128,6 +128,21 @@ void goodfoop()
     d = fma(d, d, d);
 }
 
+void bbextBad()
+{
+    gl_BoundingBoxEXT;  // ERROR without GL_EXT_primitive_bounding_box
+    gl_BoundingBox;  // ERROR, version < 320
+}
+
+#extension GL_EXT_primitive_bounding_box : enable
+
+void bbext()
+{
+    gl_BoundingBoxEXT[0] = vec4(0.0);
+    gl_BoundingBoxEXT[1] = vec4(1.0);
+    gl_BoundingBoxEXT[2] = vec4(2.0);  // ERROR, overflow
+}
+
 void bbBad()
 {
     gl_BoundingBoxOES;  // ERROR without GL_OES_primitive_bounding_box 

Test/320.tesc

@@ -118,9 +118,20 @@ void goodfoop()
 
 void bb()
 {
-    gl_BoundingBoxOES[0] = vec4(0.0);
-    gl_BoundingBoxOES[1] = vec4(1.0);
-    gl_BoundingBoxOES[2] = vec4(2.0);  // ERROR, overflow
+    gl_BoundingBoxEXT[0] = vec4(0.0); // ERROR without GL_EXT_primitive_bounding_box 
+    gl_BoundingBoxOES[0] = vec4(0.0); // ERROR without GL_OES_primitive_bounding_box 
+    gl_BoundingBox[0] = vec4(1.0);
+    gl_BoundingBox[1] = vec4(1.0);
+    gl_BoundingBox[2] = vec4(2.0);  // ERROR, overflow
+}
+
+#extension GL_EXT_primitive_bounding_box : enable
+#extension GL_OES_primitive_bounding_box : enable
+
+void bbext()
+{
+    gl_BoundingBoxEXT[1] = vec4(0.0);
+    gl_BoundingBoxOES[1] = vec4(0.0);
 }
 
 out patch badpatchBName {  // ERROR, array size required