Commit 4259a90b authored by Jurek, Pawel's avatar Jurek, Pawel Committed by gbsbuild

Changing VME types to opaque.

In new Clang versions VME types are built-in. They will be declared as opaque structure types. This change prepares IGC for this change without breaking compatibility with Clang based on LLVM4.
This is achieved by adding conversion helpers that are resolved on LLVM level.

In CShader.cpp optimisation for GRF size of VME payload is reverted, as it was not fully functionally correct. This will be addressed in future commits

Change-Id: I5f085adf9e9732da5a3511c98f9f23e25ba7e528
parent 3246f772
......@@ -1490,27 +1490,17 @@ SPIRVToLLVM::transType(SPIRVType *T) {
}
default: {
auto OC = T->getOpCode();
if (isOpaqueGenericTypeOpCode(OC))
if (isOpaqueGenericTypeOpCode(OC) ||
isSubgroupAvcINTELTypeOpCode(OC))
{
auto name = BuiltinOpaqueGenericTypeOpCodeMap::rmap(OC);
auto name = isSubgroupAvcINTELTypeOpCode(OC) ?
OCLSubgroupINTELTypeOpCodeMap::rmap(OC) :
BuiltinOpaqueGenericTypeOpCodeMap::rmap(OC);
auto *pST = M->getTypeByName(name);
pST = pST ? pST : StructType::create(*Context, name);
return mapType(T, PointerType::get(pST, getOCLOpaqueTypeAddrSpace(OC)));
}
else if (isSubgroupAvcINTELTypeOpCode(OC)) {
StructType* avcType = nullptr;
std::string typeName = OCLSubgroupINTELTypeOpCodeMap::rmap(T->getOpCode());
unsigned vectorWidth = getSubgroupAvcINTELTypeVectorWidth(OC);
if (vectorWidth == 1) {
avcType = StructType::create({ Type::getInt32Ty(*Context) }, typeName);
}
else {
spirv_assert(vectorWidth > 1);
avcType = StructType::create({ VectorType::get(Type::getInt32Ty(*Context), vectorWidth) }, typeName);
}
return mapType(T, avcType);
}
llvm_unreachable("Not implemented");
}
}
......@@ -3140,6 +3130,22 @@ SPIRVToLLVM::transSPIRVBuiltinFromInst(SPIRVInstruction *BI, BasicBlock *BB) {
std::string builtinName(getSPIRVBuiltinName(OC, BI, ArgTys, suffix));
// Fix mangling of VME builtins.
if (isIntelVMEOpCode(OC)) {
decltype(ArgTys) ArgTysWithVME_WA;
for (auto t : ArgTys) {
if (isa<PointerType>(t) &&
t->getPointerElementType()->isStructTy()) {
ArgTysWithVME_WA.push_back(t->getPointerElementType());
}
else {
ArgTysWithVME_WA.push_back(t);
}
}
builtinName = getSPIRVBuiltinName(OC, BI, ArgTysWithVME_WA, suffix);
}
if (hasReturnTypeInTypeList)
{
ArgTys.erase(ArgTys.begin());
......
......@@ -232,6 +232,11 @@ inline bool isIntelSubgroupOpCode(Op OpCode) {
return OpSubgroupShuffleINTEL <= OC && OC <= OpSubgroupImageMediaBlockWriteINTEL;
}
inline bool isIntelVMEOpCode(Op OpCode) {
unsigned OC = OpCode;
return OpVmeImageINTEL <= OC && OC <= OpSubgroupAvcSicGetInterRawSadsINTEL;
}
}
#endif /* SPIRVOPCODE_HPP_ */
......@@ -690,7 +690,7 @@ class SPIRVTypeSubgroupINTEL;
template<> inline void
SPIRVMap<std::string, Op, SPIRVTypeSubgroupINTEL>::init() {
#define _SPIRV_OP(x,y) \
add("opencl.intel_sub_group_avc_"#x, OpTypeAvc##y##INTEL);
add("struct.intel_sub_group_avc_"#x, OpTypeAvc##y##INTEL);
_SPIRV_OP(mce_payload_t, McePayload)
_SPIRV_OP(mce_result_t, MceResult)
_SPIRV_OP(sic_payload_t, SicPayload)
......
......@@ -30,6 +30,33 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
/*****************************************************************************/
#include "../../../Implementation/IGCBiF_Intrinsics.cl"
// VME helper functions - conversion to and from opaque types.
intel_sub_group_avc_mce_payload_t __builtin_IB_vme_helper_get_as_avc_mce_payload_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(intel_sub_group_avc_mce_payload_t);
intel_sub_group_avc_ime_payload_t __builtin_IB_vme_helper_get_as_avc_ime_payload_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(intel_sub_group_avc_ime_payload_t);
intel_sub_group_avc_ref_payload_t __builtin_IB_vme_helper_get_as_avc_ref_payload_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_ref_payload_t(intel_sub_group_avc_ref_payload_t);
intel_sub_group_avc_sic_payload_t __builtin_IB_vme_helper_get_as_avc_sic_payload_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_sic_payload_t(intel_sub_group_avc_sic_payload_t);
intel_sub_group_avc_mce_result_t __builtin_IB_vme_helper_get_as_avc_mce_result_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_mce_result_t(intel_sub_group_avc_mce_result_t);
intel_sub_group_avc_ime_result_t __builtin_IB_vme_helper_get_as_avc_ime_result_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_ime_result_t(intel_sub_group_avc_ime_result_t);
intel_sub_group_avc_ime_result_single_reference_streamout_t __builtin_IB_vme_helper_get_as_avc_ime_result_single_reference_streamout_t(uint8);
uint8 __builtin_IB_vme_helper_get_handle_avc_ime_result_single_reference_streamout_t(intel_sub_group_avc_ime_result_single_reference_streamout_t);
intel_sub_group_avc_ime_result_dual_reference_streamout_t __builtin_IB_vme_helper_get_as_avc_ime_result_dual_reference_streamout_t(uint8);
uint8 __builtin_IB_vme_helper_get_handle_avc_ime_result_dual_reference_streamout_t(intel_sub_group_avc_ime_result_dual_reference_streamout_t);
intel_sub_group_avc_ime_single_reference_streamin_t __builtin_IB_vme_helper_get_as_avc_ime_single_reference_streamin_t(uint);
uint __builtin_IB_vme_helper_get_handle_avc_ime_single_reference_streamin_t(intel_sub_group_avc_ime_single_reference_streamin_t);
intel_sub_group_avc_ime_dual_reference_streamin_t __builtin_IB_vme_helper_get_as_avc_ime_dual_reference_streamin_t(uint2);
uint2 __builtin_IB_vme_helper_get_handle_avc_ime_dual_reference_streamin_t(intel_sub_group_avc_ime_dual_reference_streamin_t);
intel_sub_group_avc_ref_result_t __builtin_IB_vme_helper_get_as_avc_ref_result_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_ref_result_t(intel_sub_group_avc_ref_result_t);
intel_sub_group_avc_sic_result_t __builtin_IB_vme_helper_get_as_avc_sic_result_t(uint4);
uint4 __builtin_IB_vme_helper_get_handle_avc_sic_result_t(intel_sub_group_avc_sic_result_t);
// defines
#define UNIVERSAL_INPUT_MESSAGE_NUM_GRFS 4
......@@ -597,7 +624,7 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_mo
uchar cost_precision,
intel_sub_group_avc_mce_payload_t payload )
{
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
// Set Cost Center Delta (M3.0-M3.7)
const uint FWDCostCenter = (uint)packed_cost_center_delta;
......@@ -620,7 +647,7 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_mo
const uint NonSkipZMvAdded = intel_get_message_phase_dw(handle, 1, 7) | 0x20;
handle = intel_set_message_phase_dw(handle, 1, 7, NonSkipZMvAdded);
intel_sub_group_avc_mce_payload_t result = { handle };
intel_sub_group_avc_mce_payload_t result = __builtin_IB_vme_helper_get_as_avc_mce_payload_t( handle );
return result;
}
......@@ -705,12 +732,12 @@ Description:
intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_ac_only_haar_intel_sub_group_avc_mce_payload_t(
intel_sub_group_avc_mce_payload_t payload )
{
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
const uint AConlyHAAR = intel_get_message_phase_dw(handle, 1, 7) | (1<<21);
handle = intel_set_message_phase_dw(handle, 1, 7, AConlyHAAR);
intel_sub_group_avc_mce_payload_t result = { handle };
intel_sub_group_avc_mce_payload_t result = __builtin_IB_vme_helper_get_as_avc_mce_payload_t( handle );
return result;
}
......@@ -733,7 +760,7 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_so
uchar src_field_polarity,
intel_sub_group_avc_mce_payload_t payload )
{
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
uint val = intel_get_message_phase_dw(handle, 0, 3);
val |= (0x1 << 6);
......@@ -744,7 +771,7 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_so
val |= (src_field_polarity << 19);
handle = intel_set_message_phase_dw(handle, 1, 7, val);
intel_sub_group_avc_mce_payload_t npayload = { handle };
intel_sub_group_avc_mce_payload_t npayload = __builtin_IB_vme_helper_get_as_avc_mce_payload_t( handle );
return npayload;
}
......@@ -771,11 +798,12 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_si
ref_field_polarity |= ( ref_field_polarity << 1 );
ref_field_polarity |= ( ref_field_polarity << 2 );
uint4 p = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
uint4 handle = intel_sub_group_payload_set_ref_id_polarities_raw(
ref_field_polarity,
payload.x);
p);
intel_sub_group_avc_mce_payload_t npayload = { handle };
intel_sub_group_avc_mce_payload_t npayload = __builtin_IB_vme_helper_get_as_avc_mce_payload_t( handle );
return npayload;
}
......@@ -800,11 +828,12 @@ intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_mce_set_du
uchar ref_field_polarity =
( fwd_ref_field_polarity << 0 ) | ( bwd_ref_field_polarity << 4 );
uint4 p = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
uint4 handle = intel_sub_group_payload_set_ref_id_polarities_raw(
ref_field_polarity,
payload.x);
p);
intel_sub_group_avc_mce_payload_t npayload = { handle };
intel_sub_group_avc_mce_payload_t npayload = __builtin_IB_vme_helper_get_as_avc_mce_payload_t( handle );
return npayload;
}
......@@ -826,7 +855,8 @@ Description:
ulong __builtin_spirv_intel_sub_group_avc_mce_get_motion_vectors_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
const ulong MVb = intel_simd_get_message_phase_uq(result.x, 1, 4);
uint4 r = __builtin_IB_vme_helper_get_handle_avc_mce_result_t(result);
const ulong MVb = intel_simd_get_message_phase_uq(r, 1, 4);
return MVb;
}
......@@ -846,7 +876,7 @@ Description:
ushort __builtin_spirv_intel_sub_group_avc_mce_get_inter_distortions_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
return intel_simd_get_message_phase_uw(result.x, 5, 1);
return intel_simd_get_message_phase_uw(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 5, 1);
}
INLINE ushort OVERLOADABLE
......@@ -865,7 +895,7 @@ Description:
ushort __builtin_spirv_intel_sub_group_avc_mce_get_best_inter_distortion_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
return intel_get_message_phase_uw(result.x, 0, 2*2);
return intel_get_message_phase_uw(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 0, 2*2);
}
INLINE ushort OVERLOADABLE
......@@ -884,7 +914,7 @@ Description:
uchar __builtin_spirv_intel_sub_group_avc_mce_get_inter_major_shape_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
const uchar InterMbMode = intel_get_message_phase_ub(result.x, 0, 0) & 0x3;
const uchar InterMbMode = intel_get_message_phase_ub(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 0, 0) & 0x3;
return InterMbMode;
}
......@@ -904,7 +934,7 @@ Description:
uchar __builtin_spirv_intel_sub_group_avc_mce_get_inter_minor_shapes_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
const uchar SubMbShape = intel_get_message_phase_ub(result.x, 0, 6*4+1);
const uchar SubMbShape = intel_get_message_phase_ub(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 0, 6*4+1);
return SubMbShape;
}
......@@ -924,7 +954,7 @@ Description:
uchar __builtin_spirv_intel_sub_group_avc_mce_get_inter_directions_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
const uchar SubMbPredMode = intel_get_message_phase_ub(result.x, 0, 6*4+2);
const uchar SubMbPredMode = intel_get_message_phase_ub(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 0, 6*4+2);
return SubMbPredMode;
}
......@@ -944,7 +974,7 @@ Description:
uchar __builtin_spirv_intel_sub_group_avc_mce_get_inter_motion_vector_count_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
return intel_get_message_phase_ub(result.x, 0, 0*4+3) & 0x1F;
return intel_get_message_phase_ub(__builtin_IB_vme_helper_get_handle_avc_mce_result_t(result), 0, 0*4+3) & 0x1F;
}
INLINE uchar OVERLOADABLE
......@@ -1075,7 +1105,7 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_ime_initia
const uint immValue = (partition_mask << 24) | (sad_adjustment << 20);
payload = intel_set_message_phase_dw(payload, 0, 3, immValue);
intel_sub_group_avc_ime_payload_t result = { payload };
intel_sub_group_avc_ime_payload_t result = __builtin_IB_vme_helper_get_as_avc_ime_payload_t(payload);
return result;
}
......@@ -1110,7 +1140,7 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_avc_ime_set_reference_v2i16_v2
bool multiRef,
intel_sub_group_avc_ime_payload_t payload )
{
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
// Set MaxNumMVs M1.1[5:0]
handle = intel_set_message_phase_ub(handle, 1, 1*4, 0x20);
......@@ -1348,7 +1378,7 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_avc_ime_set_reference_v2i16_v2
handle = intel_set_message_phase_dw(handle, 0, 3, SearchCtrl);
}
intel_sub_group_avc_ime_payload_t result = { handle };
intel_sub_group_avc_ime_payload_t result = __builtin_IB_vme_helper_get_as_avc_ime_payload_t(handle);
return result;
}
......@@ -1409,11 +1439,12 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_ime_set_ma
uchar max_motion_vector_count,
intel_sub_group_avc_ime_payload_t payload )
{
uchar curr = intel_get_message_phase_ub(payload.x, 1, 1*4);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uchar curr = intel_get_message_phase_ub(handle, 1, 1*4);
curr &= ~0x3F;
curr |= max_motion_vector_count;
payload.x = intel_set_message_phase_ub(payload.x, 1, 1*4, curr);
handle = intel_set_message_phase_ub(handle, 1, 1*4, curr);
payload = __builtin_IB_vme_helper_get_as_avc_ime_payload_t(handle);
return payload;
}
......@@ -1428,9 +1459,8 @@ intel_sub_group_avc_ime_set_max_motion_vector_count(
intel_sub_group_avc_mce_payload_t __builtin_spirv_intel_sub_group_avc_ime_convert_to_mce_payload_intel_sub_group_avc_ime_payload_t(
intel_sub_group_avc_ime_payload_t payload )
{
intel_sub_group_avc_mce_payload_t wrap;
wrap.x = payload.x;
return wrap;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
return __builtin_IB_vme_helper_get_as_avc_mce_payload_t(handle);
}
INLINE intel_sub_group_avc_mce_payload_t OVERLOADABLE
......@@ -1443,9 +1473,8 @@ intel_sub_group_avc_ime_convert_to_mce_payload(
intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_mce_convert_to_ime_payload_intel_sub_group_avc_mce_payload_t(
intel_sub_group_avc_mce_payload_t payload )
{
intel_sub_group_avc_ime_payload_t wrap;
wrap.x = payload.x;
return wrap;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_mce_payload_t(payload);
return __builtin_IB_vme_helper_get_as_avc_ime_payload_t(handle);
}
INLINE intel_sub_group_avc_ime_payload_t OVERLOADABLE
......@@ -1476,10 +1505,11 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_evaluat
long bwd_ref_image = getVMEImage(bwd_ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
payload.x = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, payload.x);
uint4 res = intel_vme_send_ime_new_uint4(payload.x, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_DISABLE);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
handle = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, handle);
uint4 res = intel_vme_send_ime_new_uint4(handle, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_DISABLE);
intel_sub_group_avc_ime_result_t result = { res };
intel_sub_group_avc_ime_result_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_t( res );
return result;
}
......@@ -1511,10 +1541,11 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_evaluat
long ref_image = getVMEImage(ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
payload.x = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, payload.x);
uint4 res = intel_vme_send_ime_new_uint4(payload.x, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_DISABLE);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
handle = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, handle);
uint4 res = intel_vme_send_ime_new_uint4(handle, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_DISABLE);
intel_sub_group_avc_ime_result_t result = { res };
intel_sub_group_avc_ime_result_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_t(res);
return result;
}
......@@ -1543,10 +1574,11 @@ intel_sub_group_avc_ime_result_single_reference_streamout_t __builtin_spirv_inte
long ref_image = getVMEImage(ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
payload.x = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, payload.x);
uint8 res = intel_vme_send_ime_new_uint8(payload.x, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_OUT);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
handle = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, handle);
uint8 res = intel_vme_send_ime_new_uint8(handle, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_OUT);
intel_sub_group_avc_ime_result_single_reference_streamout_t result = { res };
intel_sub_group_avc_ime_result_single_reference_streamout_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_single_reference_streamout_t(res);
return result;
}
......@@ -1577,10 +1609,11 @@ intel_sub_group_avc_ime_result_dual_reference_streamout_t __builtin_spirv_intel_
long bwd_ref_image = getVMEImage(bwd_ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
payload.x = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, payload.x);
uint8 res = intel_vme_send_ime_new_uint8(payload.x, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_OUT);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
handle = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, handle);
uint8 res = intel_vme_send_ime_new_uint8(handle, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_OUT);
intel_sub_group_avc_ime_result_dual_reference_streamout_t result = { res };
intel_sub_group_avc_ime_result_dual_reference_streamout_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_dual_reference_streamout_t( res );
return result;
}
......@@ -1613,15 +1646,17 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_evaluat
long ref_image = getVMEImage(ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uint handle_streamin = __builtin_IB_vme_helper_get_handle_avc_ime_single_reference_streamin_t(streamin);
// Copy streamin to payload IME2 and IME3
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, intel_get_message_phase(streamin.x, 0));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+3, intel_get_message_phase(streamin.x, 1));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, intel_get_message_phase(handle_streamin, 0));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+3, intel_get_message_phase(handle_streamin, 1));
handle = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, handle);
uint4 res = intel_vme_send_ime_new_uint4(handle, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_IN);
intel_sub_group_avc_ime_result_t result = { res };
intel_sub_group_avc_ime_result_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_t(res);
return result;
}
......@@ -1656,21 +1691,24 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_evaluat
uint8 newPayload = __builtin_IB_create_message_phases_no_init_uint8(16);
uint4 handle_p = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uint2 handle_s = __builtin_IB_vme_helper_get_handle_avc_ime_dual_reference_streamin_t(streamin);
for (int i=0; i < UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2; i++)
{
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(payload.x, i));
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(handle_p, i));
}
// Copy streamin to payload IME2 ~ IME5
for (int i=UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, j=0; i < 10; i++, j++)
{
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(streamin.x, j));
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(handle_s, j));
}
newPayload = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, newPayload);
uint4 res = intel_vme_send_ime_new_uint4(newPayload, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_IN);
intel_sub_group_avc_ime_result_t result = { res };
intel_sub_group_avc_ime_result_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_t( res );
return result;
}
......@@ -1704,15 +1742,16 @@ intel_sub_group_avc_ime_result_single_reference_streamout_t __builtin_spirv_inte
long ref_image = getVMEImage(ref_image_vme);
long vme_accelerator = getVMESampler(src_image_vme);
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uint handle_s = __builtin_IB_vme_helper_get_handle_avc_ime_single_reference_streamin_t(streamin);
// Copy streamin to payload IME2 and IME3
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, intel_get_message_phase(streamin.x, 0));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+3, intel_get_message_phase(streamin.x, 1));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, intel_get_message_phase(handle_s, 0));
handle = intel_set_message_phase(handle, UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+3, intel_get_message_phase(handle_s, 1));
handle = intel_sub_group_payload_set_single_ref_id(src_image, ref_image, handle);
uint8 res = intel_vme_send_ime_new_uint8(handle, src_image, ref_image, ref_image, vme_accelerator, VME_STREAM_INOUT);
intel_sub_group_avc_ime_result_single_reference_streamout_t result = { res };
intel_sub_group_avc_ime_result_single_reference_streamout_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_single_reference_streamout_t( res );
return result;
}
......@@ -1747,22 +1786,25 @@ intel_sub_group_avc_ime_result_dual_reference_streamout_t __builtin_spirv_intel_
uint8 newPayload = __builtin_IB_create_message_phases_no_init_uint8(16);
uint4 handle_p = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uint2 handle_s = __builtin_IB_vme_helper_get_handle_avc_ime_dual_reference_streamin_t(streamin);
for (int i=0; i < UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2; i++)
{
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(payload.x, i));
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(handle_p, i));
}
// Copy streamin to payload IME2 ~ IME5
for (int i=UNIVERSAL_INPUT_MESSAGE_NUM_GRFS+2, j=0; i < 10; i++, j++)
{
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(streamin.x, j));
newPayload = intel_set_message_phase(newPayload, i, intel_get_message_phase(handle_s, j));
}
newPayload = intel_sub_group_payload_set_dual_ref_id(src_image, fwd_ref_image, bwd_ref_image, newPayload);
uint8 res = intel_vme_send_ime_new_uint8(newPayload, src_image, fwd_ref_image, bwd_ref_image, vme_accelerator, VME_STREAM_INOUT);
intel_sub_group_avc_ime_result_dual_reference_streamout_t result = { res };
intel_sub_group_avc_ime_result_dual_reference_streamout_t result = __builtin_IB_vme_helper_get_as_avc_ime_result_dual_reference_streamout_t( res );
return result;
}
......@@ -1790,12 +1832,13 @@ intel_sub_group_avc_ime_single_reference_streamin_t __builtin_spirv_intel_sub_gr
intel_sub_group_avc_ime_result_single_reference_streamout_t result )
{
uint handle = __builtin_IB_create_message_phases_no_init(2);
uint8 handle_r = __builtin_IB_vme_helper_get_handle_avc_ime_result_single_reference_streamout_t(result);
// Copy IME2 from result W7
handle = intel_set_message_phase(handle, 0, intel_get_message_phase(result.x, RETURN_MESSAGE_NUM_GRFS));
handle = intel_set_message_phase(handle, 0, intel_get_message_phase(handle_r, RETURN_MESSAGE_NUM_GRFS));
// Copy IME3 from result W8
handle = intel_set_message_phase(handle, 1, intel_get_message_phase(result.x, RETURN_MESSAGE_NUM_GRFS + 1));
handle = intel_set_message_phase(handle, 1, intel_get_message_phase(handle_r, RETURN_MESSAGE_NUM_GRFS + 1));
intel_sub_group_avc_ime_single_reference_streamin_t nresult = { handle };
intel_sub_group_avc_ime_single_reference_streamin_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_single_reference_streamin_t(handle);
return nresult;
}
......@@ -1810,12 +1853,13 @@ intel_sub_group_avc_ime_dual_reference_streamin_t __builtin_spirv_intel_sub_grou
intel_sub_group_avc_ime_result_dual_reference_streamout_t result )
{
uint2 handle = __builtin_IB_create_message_phases_no_init_uint2(4);
uint8 handle_r = __builtin_IB_vme_helper_get_handle_avc_ime_result_dual_reference_streamout_t(result);
// Copy IME2~IME5 from result W7~W10
for (int i = 0; i < 4; i++)
{
handle = intel_set_message_phase(handle, i, intel_get_message_phase(result.x, RETURN_MESSAGE_NUM_GRFS+i));
handle = intel_set_message_phase(handle, i, intel_get_message_phase(handle_r, RETURN_MESSAGE_NUM_GRFS+i));
}
intel_sub_group_avc_ime_dual_reference_streamin_t nresult = { handle };
intel_sub_group_avc_ime_dual_reference_streamin_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_dual_reference_streamin_t(handle);
return nresult;
}
......@@ -1830,12 +1874,13 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_strip_s
intel_sub_group_avc_ime_result_single_reference_streamout_t result )
{
uint4 res = __builtin_IB_create_message_phases_no_init_uint4(RETURN_MESSAGE_NUM_GRFS+1);
uint8 handle_r = __builtin_IB_vme_helper_get_handle_avc_ime_result_single_reference_streamout_t(result);
for (int i = 0; i < RETURN_MESSAGE_NUM_GRFS; i++) {
res = intel_set_message_phase(res, i, intel_get_message_phase(result.x, i));
res = intel_set_message_phase(res, i, intel_get_message_phase(handle_r, i));
}
intel_sub_group_avc_ime_result_t nresult = { res };
intel_sub_group_avc_ime_result_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_result_t(res);
return nresult;
}
......@@ -1850,12 +1895,13 @@ intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_ime_strip_d
intel_sub_group_avc_ime_result_dual_reference_streamout_t result )
{
uint4 res = __builtin_IB_create_message_phases_no_init_uint4(RETURN_MESSAGE_NUM_GRFS + 1);
uint8 handle_r = __builtin_IB_vme_helper_get_handle_avc_ime_result_dual_reference_streamout_t(result);
for (int i = 0; i < RETURN_MESSAGE_NUM_GRFS; i++) {
res = intel_set_message_phase(res, i, intel_get_message_phase(result.x, i));
res = intel_set_message_phase(res, i, intel_get_message_phase(handle_r, i));
}
intel_sub_group_avc_ime_result_t nresult = { res };
intel_sub_group_avc_ime_result_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_result_t(res);
return nresult;
}
......@@ -1881,27 +1927,28 @@ uint __builtin_spirv_intel_sub_group_avc_ime_get_streamout_major_shape_motion_ve
uint retValue = 0;
// IME Streamout follows the same format as the IME Streamin message phases (IME2-IME5).
const uint reg = (direction == CLK_AVC_ME_MAJOR_FORWARD_INTEL) ? /*fwd*/(RETURN_MESSAGE_NUM_GRFS+1) : /*bwd*/(RETURN_MESSAGE_NUM_GRFS+3);
uint8 handle_r = __builtin_IB_vme_helper_get_handle_avc_ime_result_dual_reference_streamout_t(result);
if (shape == VME_MAJOR_16x16) {
// WX+2.5 fwd
// WX+4.5 bwd
const uint reg = (direction == CLK_AVC_ME_MAJOR_FORWARD_INTEL) ? /*fwd*/(RETURN_MESSAGE_NUM_GRFS) : /*bwd*/(RETURN_MESSAGE_NUM_GRFS+2);
retValue = intel_get_message_phase_dw(result.x, reg, 5);
retValue = intel_get_message_phase_dw(handle_r, reg, 5);
}
else if (shape == VME_MAJOR_16x8) {
// WX+3.0 and WX+3.1 fwd
// WX+5.0 and WX+5.1 bwd
retValue = intel_broadcast_message_phase_dw(result.x, reg, 0, 2);
retValue = intel_broadcast_message_phase_dw(handle_r, reg, 0, 2);
}
else if (shape == VME_MAJOR_8x16) {
// WX+3.2 and WX+3.3
// WX+5.2 and WX+5.3
retValue = intel_broadcast_message_phase_dw(result.x, reg, 2, 2);
retValue = intel_broadcast_message_phase_dw(handle_r, reg, 2, 2);
}
else if (shape == VME_MAJOR_8x8) {
// WX+3.4 ~ WX+3.7
// WX+5.4 ~ WX+5.7
retValue = intel_broadcast_message_phase_dw(result.x, reg, 4, 4);
retValue = intel_broadcast_message_phase_dw(handle_r, reg, 4, 4);
}
return retValue;
......@@ -1920,8 +1967,8 @@ uint __builtin_spirv_intel_sub_group_avc_ime_get_streamout_major_shape_motion_ve
intel_sub_group_avc_ime_result_single_reference_streamout_t result,
uchar major_shape )
{
intel_sub_group_avc_ime_result_dual_reference_streamout_t nresult;
nresult.x = result.x;
uint8 handle = __builtin_IB_vme_helper_get_handle_avc_ime_result_single_reference_streamout_t(result);
intel_sub_group_avc_ime_result_dual_reference_streamout_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_result_dual_reference_streamout_t(handle);
return intel_sub_group_avc_ime_get_streamout_major_shape_motion_vectors(nresult, major_shape, CLK_AVC_ME_MAJOR_FORWARD_INTEL);
}
......@@ -1946,6 +1993,7 @@ ushort __builtin_spirv_intel_sub_group_avc_ime_get_streamout_major_shape_distort
uchar direction )
{
ushort retValue = 0;
uint8 handle = __builtin_IB_vme_helper_get_handle_avc_ime_result_dual_reference_streamout_t(result);
// IME Streamout follows the same format as the IME Streamin message phases (IME2-IME5).
const uint reg = (direction == CLK_AVC_ME_MAJOR_FORWARD_INTEL) ?
/*fwd*/ RETURN_MESSAGE_NUM_GRFS :
......@@ -1954,22 +2002,22 @@ ushort __builtin_spirv_intel_sub_group_avc_ime_get_streamout_major_shape_distort
if (shape == VME_MAJOR_16x16) {
// WX+2.4 [15:0] - Rec0 Shape 16x16 Distortion
// WX+4.4 [15:0] - Rec1 Shape 16x16 Distortion
retValue = intel_get_message_phase_uw(result.x, reg, 4*2);
retValue = intel_get_message_phase_uw(handle, reg, 4*2);
}
else if (shape == VME_MAJOR_16x8) {
// WX+2.0 [15:00] - Rec0 Shape 16x8_0 Distortion
// WX+4.0 [15:00] - Rec1 Shape 16x8_0 Distortion
retValue = intel_broadcast_message_phase_uw(result.x, reg, 0, 2);
retValue = intel_broadcast_message_phase_uw(handle, reg, 0, 2);
}
else if (shape == VME_MAJOR_8x16) {
// WX+2.1 [15:00] - Rec0 Shape 8x16_0 Distortion
// WX+4.1 [15:00] - Rec1 Shape 8x16_0 Distortion
retValue = intel_broadcast_message_phase_uw(result.x, reg, 1*2, 2);
retValue = intel_broadcast_message_phase_uw(handle, reg, 1*2, 2);
}
else if (shape == VME_MAJOR_8x8) {
// WX+2.2 [15:00] - Rec0 Shape 8x8_0 Distortion
// WX+4.2 [15:00] - Rec1 Shape 8x8_0 Distortion
retValue = intel_broadcast_message_phase_uw(result.x, reg, 2*2, 4);
retValue = intel_broadcast_message_phase_uw(handle, reg, 2*2, 4);
}
return retValue;
......@@ -1988,8 +2036,8 @@ ushort __builtin_spirv_intel_sub_group_avc_ime_get_streamout_major_shape_distort
intel_sub_group_avc_ime_result_single_reference_streamout_t result,
uchar major_shape )
{
intel_sub_group_avc_ime_result_dual_reference_streamout_t nresult;
nresult.x = result.x;
uint8 handle = __builtin_IB_vme_helper_get_handle_avc_ime_result_single_reference_streamout_t(result);
intel_sub_group_avc_ime_result_dual_reference_streamout_t nresult = __builtin_IB_vme_helper_get_as_avc_ime_result_dual_reference_streamout_t(handle);
return intel_sub_group_avc_ime_get_streamout_major_shape_distortions(nresult, major_shape, 0);
}
......@@ -2004,9 +2052,8 @@ intel_sub_group_avc_ime_get_streamout_major_shape_distortions(
intel_sub_group_avc_mce_result_t __builtin_spirv_intel_sub_group_avc_ime_convert_to_mce_result_intel_sub_group_avc_ime_result_t(
intel_sub_group_avc_ime_result_t result )
{
intel_sub_group_avc_mce_result_t wrap;
wrap.x = result.x;
return wrap;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_result_t(result);
return __builtin_IB_vme_helper_get_as_avc_mce_result_t(handle);
}
INLINE intel_sub_group_avc_mce_result_t OVERLOADABLE
......@@ -2019,9 +2066,8 @@ intel_sub_group_avc_ime_convert_to_mce_result(
intel_sub_group_avc_ime_result_t __builtin_spirv_intel_sub_group_avc_mce_convert_to_ime_result_intel_sub_group_avc_mce_result_t(
intel_sub_group_avc_mce_result_t result )
{
intel_sub_group_avc_ime_result_t wrap;
wrap.x = result.x;
return wrap;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_mce_result_t(result);
return __builtin_IB_vme_helper_get_as_avc_ime_result_t(handle);
}
INLINE intel_sub_group_avc_ime_result_t OVERLOADABLE
......@@ -2040,11 +2086,11 @@ Description:
intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_ime_set_unidirectional_mix_disable_intel_sub_group_avc_ime_payload_t(
intel_sub_group_avc_ime_payload_t payload )
{
uint4 handle = payload.x;
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
const uint UniMixDis = intel_get_message_phase_dw(handle, 1, 1) | (1<<28);
handle = intel_set_message_phase_dw(handle, 1, 1, UniMixDis);
intel_sub_group_avc_ime_payload_t result = { handle };
intel_sub_group_avc_ime_payload_t result = __builtin_IB_vme_helper_get_as_avc_ime_payload_t(handle);
return result;
}
......@@ -2066,11 +2112,12 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_ime_set_ea
uchar threshold,
intel_sub_group_avc_ime_payload_t payload )
{
uchar newval = intel_get_message_phase_ub(payload.x, 1, 0*4) | (1 << 5);
payload.x = intel_set_message_phase_ub(payload.x, 1, 0*4, newval);
payload.x = intel_set_message_phase_ub(payload.x, 1, 0*4+3, threshold);
uint4 handle = __builtin_IB_vme_helper_get_handle_avc_ime_payload_t(payload);
uchar newval = intel_get_message_phase_ub(handle, 1, 0*4) | (1 << 5);
handle = intel_set_message_phase_ub(handle, 1, 0*4, newval);
handle = intel_set_message_phase_ub(handle, 1, 0*4+3, threshold);
return payload;
return __builtin_IB_vme_helper_get_as_avc_ime_payload_t(handle);
}
INLINE intel_sub_group_avc_ime_payload_t OVERLOADABLE
......@@ -2101,14 +2148,6 @@ intel_sub_group_avc_ime_payload_t __builtin_spirv_intel_sub_group_avc_ime_set_ea
return intel_sub_group_avc_ime_set_early_search_termination_threshold(threshold, payload);
}
INLINE intel_sub_group_avc_ime_payload_t OVERLOADABLE
intel_sub_group_avc_ime_set_early_unidirectional_search_termination_threshold(
uchar threshold,
intel_sub_group_avc_ime_payload_t payload )
{