Integrate LLVM at 8885b5c0626065274cb8f8a634d45779a0f6ff2b

lib/Conversion/TorchToTosa/TorchToTosa.cpp

@@ -1848,22 +1848,52 @@ class ConvertAtenMatmulBaseOp : public OpConversionPattern<AtenOpT> {
     SmallVector<int64_t> matmulOutputShape(
         {matmulLhsShape[0], matmulLhsShape[1], matmulRhsShape[2]});
     Type outputElemTy;
-    if (isa<mlir::FloatType>(lhsElemTy)) {
-      outputElemTy = lhsElemTy;
-    } else { // qint8 emits i32 matmul output
+
+    bool isInputElemTyQInt8 = false;
+    if (isa<mlir::quant::UniformQuantizedType>(lhsElemTy)) {
+      mlir::quant::UniformQuantizedType inputQTy =
+          dyn_cast<mlir::quant::UniformQuantizedType>(lhsElemTy);
+      if (inputQTy.getStorageTypeIntegralWidth() == 8)
+        isInputElemTyQInt8 = true;
+    }
+
+    if (isInputElemTyQInt8) {
+      // qint8 emits i32 matmul output
       outputElemTy = rewriter.getIntegerType(32);
+    } else {
+      outputElemTy = lhsElemTy;
     }
 
     auto mmOutputTy = RankedTensorType::get(
         makeShapeLLVMCompatible(matmulOutputShape), outputElemTy);
-    auto mmOpResult =
-        rewriter
-            .create<tosa::MatMulOp>(
-                op->getLoc(),
-                OpConversionPattern<AtenOpT>::getTypeConverter()->convertType(
-                    mmOutputTy),
-                matmulLhs, matmulRhs)
-            .getResult();
+
+    Value mmOpResult;
+    if (!isInputElemTyQInt8) {
+      // LHS and RHS tensors' zero points must be zero for non-int8 types
+      Value lhsZp =
+          tosa::createZeroPointTensor(rewriter, op->getLoc(), lhsElemTy, 0)
+              .value();
+      Value rhsZp =
+          tosa::createZeroPointTensor(rewriter, op->getLoc(), rhsElemTy, 0)
+              .value();
+      mmOpResult =
+          rewriter
+              .create<tosa::MatMulOp>(
+                  op->getLoc(),
+                  OpConversionPattern<AtenOpT>::getTypeConverter()->convertType(
+                      mmOutputTy),
+                  matmulLhs, matmulRhs, lhsZp, rhsZp)
+              .getResult();
+    } else {
+      mmOpResult =
+          rewriter
+              .create<tosa::MatMulOp>(
+                  op->getLoc(),
+                  OpConversionPattern<AtenOpT>::getTypeConverter()->convertType(
+                      mmOutputTy),
+                  matmulLhs, matmulRhs)
+              .getResult();
+    }
 
     // Perform the reshape to output shape. This is always required unless max
     // input rank=3 and there was no broadcasting, in which case the tosa.matmul

lib/Conversion/TorchToTosa/TosaLegalizeUtils.cpp

@@ -15,6 +15,21 @@
 namespace mlir {
 namespace tosa {
 
+Value buildRescaleMultiplier(bool scale32, PatternRewriter &rewriter,
+                             Operation *op, ArrayRef<int32_t> multipliers) {
+  if (scale32) {
+    return tosa::getConstTensor<int32_t>(
+               rewriter, op, multipliers,
+               {static_cast<int64_t>(multipliers.size())})
+        .value();
+  } else {
+    SmallVector<int16_t> vec(multipliers.begin(), multipliers.end());
+    return tosa::getConstTensor<int16_t>(rewriter, op, vec,
+                                         {static_cast<int64_t>(vec.size())})
+        .value();
+  }
+}
+
 // Create a TOSA rescale op from input framework tensor, zero points and
 // rounding mode
 Value buildRescale(PatternRewriter &rewriter, Operation *op,
@@ -28,14 +43,22 @@ Value buildRescale(PatternRewriter &rewriter, Operation *op,
 
   computeMultiplierAndShift(scale, multiplier, shift, scale_width);
 
+  Value multiplier_val =
+      buildRescaleMultiplier(scale32, rewriter, op, {multiplier});
+  auto shift_val = tosa::getConstTensor<int8_t>(
+                       rewriter, op, {static_cast<int8_t>(shift)}, {1})
+                       .value();
+
+  bool input_unsigned = input_val.getType().isUnsignedInteger();
+  bool output_unsigned = output_type.isUnsignedInteger();
+
   auto rescale_op = CreateOpAndInfer<tosa::RescaleOp>(
-      rewriter, op->getLoc(), output_type, input_val,
+      rewriter, op->getLoc(), output_type, input_val, multiplier_val, shift_val,
       rewriter.getI32IntegerAttr(static_cast<int32_t>(input_zp)),
       rewriter.getI32IntegerAttr(static_cast<int32_t>(output_zp)),
-      rewriter.getDenseI32ArrayAttr({multiplier}),
-      rewriter.getDenseI8ArrayAttr({static_cast<int8_t>(shift)}),
       rewriter.getBoolAttr(scale32), rewriter.getBoolAttr(double_round),
-      rewriter.getBoolAttr(false));
+      rewriter.getBoolAttr(false), rewriter.getBoolAttr(input_unsigned),
+      rewriter.getBoolAttr(output_unsigned));
 
   return rescale_op.getResult();
 }
@@ -70,6 +93,9 @@ Value buildRescaleOpConvOutput(PatternRewriter &rewriter, Operation *op,
   bool scale32 = isScale32(output_qtype);
   int32_t scale_width = scale32 ? 32 : 16;
 
+  bool input_unsigned = input_qtype.isUnsignedInteger();
+  bool output_unsigned = output_qtype.isUnsignedInteger();
+
   if (auto weight_per_tensor_qtype =
           dyn_cast<mlir::quant::UniformQuantizedType>(
               weight_type.getElementType())) {
@@ -83,13 +109,19 @@ Value buildRescaleOpConvOutput(PatternRewriter &rewriter, Operation *op,
 
     computeMultiplierAndShift(op_tensor_scale, multiplier, shift, scale_width);
 
+    Value multiplier_val =
+        buildRescaleMultiplier(scale32, rewriter, op, {multiplier});
+    auto shift_val = tosa::getConstTensor<int8_t>(
+                         rewriter, op, {static_cast<int8_t>(shift)}, {1})
+                         .value();
+
     auto rescale_op = CreateOpAndInfer<tosa::RescaleOp>(
-        rewriter, op->getLoc(), output_type, conv_val,
-        rewriter.getI32IntegerAttr(0), rewriter.getI32IntegerAttr(output_zp),
-        rewriter.getDenseI32ArrayAttr({multiplier}),
-        rewriter.getDenseI8ArrayAttr({static_cast<int8_t>(shift)}),
-        rewriter.getBoolAttr(scale32), rewriter.getBoolAttr(true),
-        rewriter.getBoolAttr(false));
+        rewriter, op->getLoc(), output_type, conv_val, multiplier_val,
+        shift_val, rewriter.getI32IntegerAttr(0),
+        rewriter.getI32IntegerAttr(output_zp), rewriter.getBoolAttr(scale32),
+        rewriter.getBoolAttr(true), rewriter.getBoolAttr(false),
+        rewriter.getBoolAttr(input_unsigned),
+        rewriter.getBoolAttr(output_unsigned));
 
     return rescale_op.getResult();
 
@@ -120,12 +152,20 @@ Value buildRescaleOpConvOutput(PatternRewriter &rewriter, Operation *op,
       shift_arr.push_back(static_cast<int8_t>(shift));
     }
 
+    Value multiplier_val =
+        buildRescaleMultiplier(scale32, rewriter, op, multiplier_arr);
+    auto shift_val =
+        tosa::getConstTensor<int8_t>(rewriter, op, shift_arr,
+                                     {static_cast<int64_t>(shift_arr.size())})
+            .value();
+
     auto rescale_op = CreateOpAndInfer<tosa::RescaleOp>(
-        rewriter, op->getLoc(), output_type, conv_val,
-        rewriter.getI32IntegerAttr(0), rewriter.getI32IntegerAttr(output_zp),
-        rewriter.getDenseI32ArrayAttr(multiplier_arr),
-        rewriter.getDenseI8ArrayAttr(shift_arr), rewriter.getBoolAttr(scale32),
-        rewriter.getBoolAttr(true), rewriter.getBoolAttr(true));
+        rewriter, op->getLoc(), output_type, conv_val, multiplier_val,
+        shift_val, rewriter.getI32IntegerAttr(0),
+        rewriter.getI32IntegerAttr(output_zp), rewriter.getBoolAttr(scale32),
+        rewriter.getBoolAttr(true), rewriter.getBoolAttr(true),
+        rewriter.getBoolAttr(input_unsigned),
+        rewriter.getBoolAttr(output_unsigned));
 
     return rescale_op.getResult();
 
@@ -408,6 +448,10 @@ template std::optional<Value>
 getConstTensor<int8_t>(PatternRewriter &, Operation *, ArrayRef<int8_t> vec,
                        ArrayRef<int64_t> shape, std::optional<Type> dtype);
 
+template std::optional<Value>
+getConstTensor<int16_t>(PatternRewriter &, Operation *, ArrayRef<int16_t> vec,
+                        ArrayRef<int64_t> shape, std::optional<Type> dtype);
+
 template std::optional<Value>
 getConstTensor<int32_t>(PatternRewriter &, Operation *, ArrayRef<int32_t> vec,
                         ArrayRef<int64_t> shape, std::optional<Type> dtype);

projects/pt1/e2e_testing/xfail_sets.py

@@ -3472,7 +3472,6 @@
     "AtenMatmulQint8VM_basic",
     "AtenMatmulQint8VV_basic",
     "AtenMatmulQint8_basic",
-    "AtenMmIntTypes_basic",
     "AtenMmQMixedSigni8_basic",
     "AtenMmQint8_basic",
     "AtenMmQuint8_basic",
@@ -3496,7 +3495,6 @@
     "BincountMinlengthModule_basic",
     "BincountModule_basic",
     "BincountStaticSizeModule_basic",
-    "BmmIntModule_basic",
     "BoolFloatConstantModule_basic",
     "BoolFloatFalseModule_basic",
     "BoolFloatTrueModule_basic",