Improve buy/sell detection in sub-optimal conditions

Author: EtienneLamoureux

CHANGELOG.md

@@ -1,4 +1,11 @@
 # SC Trade Companion
+## 1.0.1
+### Bugs
+- Improve buy/sell detection in sub-optimal conditions
+
+### Other
+- Add an INFO level logs when listings are sent to the website
+
 ## 1.0.0
 ### Features
 - Spellcheck and auto-correct location and commodity names

build.gradle

@@ -5,7 +5,7 @@ plugins {
 }
 
 group = 'tools.sctrade'
-version = '1.0.0'
+version = '1.0.1'
 
 java {
 	sourceCompatibility = '17'

src/main/java/tools/sctrade/companion/domain/commodity/CommodityListingFactory.java

@@ -51,7 +51,7 @@ public Collection<CommodityListing> build(BufferedImage screenCapture, String lo
       return buildCommodityListings(location, transactionType, rawListings, batchId);
     } catch (Exception e) {
       logger.error("Error while reading listings", e);
-
+      // TODO
       return Collections.emptyList();
     } finally {
       listingsOcr.remove();

src/main/java/tools/sctrade/companion/domain/commodity/TransactionTypeExtractor.java

@@ -2,6 +2,7 @@
 
 import java.awt.Rectangle;
 import java.awt.image.BufferedImage;
+import java.util.ArrayList;
 import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -10,11 +11,14 @@
 import tools.sctrade.companion.domain.ocr.LocatedFragment;
 import tools.sctrade.companion.domain.ocr.OcrResult;
 import tools.sctrade.companion.domain.ocr.OcrUtil;
+import tools.sctrade.companion.exceptions.UnreadableTransactionTypeException;
 import tools.sctrade.companion.utils.ImageUtil;
 
 public class TransactionTypeExtractor {
   private final Logger logger = LoggerFactory.getLogger(TransactionTypeExtractor.class);
   private static final String SHOP_INVENTORY = "shop inventory";
+  private static final String BUY = "buy";
+  private static final Object SELL = "sell";
 
   private ImageWriter imageWriter;
 
@@ -23,12 +27,16 @@ public TransactionTypeExtractor(ImageWriter imageWriter) {
   }
 
   public TransactionType extract(BufferedImage screenCapture, OcrResult result) {
-    screenCapture = cropAroundButtons(screenCapture, result);
+    screenCapture = ImageUtil.crop(screenCapture, new Rectangle((screenCapture.getWidth() / 2), 0,
+        (screenCapture.getWidth() - (screenCapture.getWidth() / 2)), screenCapture.getHeight()));
+    screenCapture = ImageUtil.makeGreyscaleCopy(screenCapture);
+    var buttonAreaBoundingBox = findButtonAreaBoundingBox(screenCapture, result);
+    screenCapture = ImageUtil.crop(screenCapture, buttonAreaBoundingBox);
     var boundingBoxes = getButtonBoundingBoxes(screenCapture);
 
     boundingBoxes
         .sort((n, m) -> Double.compare(m.getWidth() * m.getHeight(), n.getWidth() * n.getHeight()));
-    var boundingBoxesOrderedByX = boundingBoxes.subList(0, 2);
+    var boundingBoxesOrderedByX = new ArrayList<>(boundingBoxes.subList(0, 2));
     boundingBoxesOrderedByX.sort((n, m) -> Double.compare(n.getMinX(), m.getMinX()));
 
     Rectangle buyRectangle = boundingBoxesOrderedByX.get(0);
@@ -39,29 +47,33 @@ public TransactionType extract(BufferedImage screenCapture, OcrResult result) {
     var sellImage = ImageUtil.crop(screenCapture, sellRectangle);
     imageWriter.write(sellImage, ImageType.SELL_BUTTON);
 
-    double buyButtonAreaOverSellButtonArea = (buyRectangle.getWidth() * buyRectangle.getHeight())
-        / (sellRectangle.getWidth() * sellRectangle.getHeight());
-
-    if (Math.abs(1 - buyButtonAreaOverSellButtonArea) >= 0.2) {
-      logger.warn("Detected buttons are not the same size: one button may have not been captured");
-      return (buyButtonAreaOverSellButtonArea > 1) ? TransactionType.SELLS : TransactionType.BUYS;
-    } else {
+    if (buttonsAreAligned(buyRectangle, sellRectangle)) {
       return extractByLuminance(buyImage, sellImage);
+    } else {
+      logger.warn("Detected buttons are not aligned: one button may have not been captured");
+      var buttonBoundingBox = getNormalizedButtonBoundingBox(buttonAreaBoundingBox, boundingBoxes);
+
+      return extractByStringPosition(result, buttonBoundingBox);
     }
   }
 
-  private BufferedImage cropAroundButtons(BufferedImage screenCapture, OcrResult result) {
+  private Rectangle findButtonAreaBoundingBox(BufferedImage screenCapture, OcrResult result) {
     Rectangle shopInventoryRectangle = getShopInventoryRectangle(result);
-    screenCapture = ImageUtil.crop(screenCapture, new Rectangle((screenCapture.getWidth() / 2), 0,
-        (screenCapture.getWidth() - (screenCapture.getWidth() / 2)), screenCapture.getHeight()));
-    Rectangle buttonsAreaRectangle = new Rectangle((int) shopInventoryRectangle.getMinX(),
+    Rectangle buttonAreaBoundingBox = new Rectangle((int) shopInventoryRectangle.getMinX(),
         (int) (shopInventoryRectangle.getMinY() - shopInventoryRectangle.getHeight()),
         (int) (shopInventoryRectangle.getWidth() * 3),
         (int) (shopInventoryRectangle.getHeight() * 5));
-    screenCapture = ImageUtil.crop(screenCapture, buttonsAreaRectangle);
-    screenCapture = ImageUtil.makeGreyscaleCopy(screenCapture);
 
-    return screenCapture;
+    return buttonAreaBoundingBox;
+  }
+
+  private Rectangle getShopInventoryRectangle(OcrResult result) {
+    var fragments = result.getColumns().parallelStream()
+        .flatMap(n -> n.getFragments().parallelStream()).toList();
+    LocatedFragment shopInventoryFragment =
+        OcrUtil.findFragmentClosestTo(fragments, SHOP_INVENTORY);
+
+    return shopInventoryFragment.getBoundingBox();
   }
 
   private List<Rectangle> getButtonBoundingBoxes(BufferedImage screenCapture) {
@@ -73,6 +85,63 @@ private List<Rectangle> getButtonBoundingBoxes(BufferedImage screenCapture) {
     return boundingBoxes;
   }
 
+  private boolean buttonsAreAligned(Rectangle buyRectangle, Rectangle sellRectangle) {
+    var yOverlap1 = sellRectangle.getMaxY() - buyRectangle.getMinY();
+    var yOverlap2 = buyRectangle.getMaxY() - sellRectangle.getMinY();
+    var yOverlap = Math.min(yOverlap1, yOverlap2);
+    var yHeight = Math.min(buyRectangle.getHeight(), sellRectangle.getHeight());
+
+    return (yOverlap / yHeight) > 0.66;
+  }
+
+  private Rectangle getNormalizedButtonBoundingBox(Rectangle buttonAreaBoundingBox,
+      List<Rectangle> boundingBoxes) {
+    var buttonBoundingBox = boundingBoxes.iterator().next();
+    buttonBoundingBox.setLocation(buttonAreaBoundingBox.x + buttonBoundingBox.x,
+        buttonAreaBoundingBox.y + buttonBoundingBox.y);
+
+    return buttonBoundingBox;
+  }
+
+  private TransactionType extractByStringPosition(OcrResult result, Rectangle buttonBoundingBox) {
+    var buttonFragments =
+        result.getColumns().parallelStream().flatMap(n -> n.getFragments().stream())
+            .filter(n -> n.getBoundingBox().intersectsLine(Double.MIN_VALUE,
+                buttonBoundingBox.getCenterY(), Double.MAX_VALUE, buttonBoundingBox.getCenterY()))
+            .toList();
+
+    var buyFragment =
+        buttonFragments.parallelStream().filter(n -> n.getText().equals(BUY)).findFirst();
+
+    if (buyFragment.isPresent()) {
+      if (buttonBoundingBox.contains(buyFragment.get().getBoundingBox())) {
+        return TransactionType.SELLS;
+      }
+
+      if (buyFragment.get().getBoundingBox().getMaxX() < buttonBoundingBox.getMinX()) {
+        // buttonBoundingBox is the SELL button
+        return TransactionType.BUYS;
+      }
+    }
+
+    var sellFragment =
+        buttonFragments.parallelStream().filter(n -> n.getText().equals(SELL)).findFirst();
+
+    if (sellFragment.isPresent()) {
+      if (buttonBoundingBox.contains(sellFragment.get().getBoundingBox())) {
+        return TransactionType.BUYS;
+      }
+
+      if (buttonBoundingBox.getMaxX() < sellFragment.get().getBoundingBox().getMinX()) {
+        // buttonBoundingBox is the BUY button
+        return TransactionType.SELLS;
+      }
+    }
+
+    logger.warn("Could not detect transaction type");
+    throw new UnreadableTransactionTypeException();
+  }
+
   private TransactionType extractByLuminance(BufferedImage buyImage, BufferedImage sellImage) {
     var buyRectangleColor = ImageUtil.calculateDominantColor(buyImage);
     var buyRectangleLuminance = buyRectangleColor.getRed();
@@ -83,13 +152,4 @@ private TransactionType extractByLuminance(BufferedImage buyImage, BufferedImage
     return (buyRectangleLuminance > sellRectangleLuminance) ? TransactionType.SELLS
         : TransactionType.BUYS;
   }
-
-  private Rectangle getShopInventoryRectangle(OcrResult result) {
-    var fragments = result.getColumns().parallelStream()
-        .flatMap(n -> n.getFragments().parallelStream()).toList();
-    LocatedFragment shopInventoryFragment =
-        OcrUtil.findFragmentClosestTo(fragments, SHOP_INVENTORY);
-
-    return shopInventoryFragment.getBoundingBox();
-  }
 }

src/main/java/tools/sctrade/companion/exceptions/UnreadableTransactionTypeException.java

@@ -0,0 +1,11 @@
+package tools.sctrade.companion.exceptions;
+
+import tools.sctrade.companion.utils.LocalizationUtil;
+
+public class UnreadableTransactionTypeException extends RuntimeException {
+  private static final long serialVersionUID = 82552490134110748L;
+
+  public UnreadableTransactionTypeException() {
+    super(LocalizationUtil.get("errorUnreadableTransactionType"));
+  }
+}

src/main/java/tools/sctrade/companion/output/commodity/ScTradeToolsClient.java

@@ -21,6 +21,7 @@
 import tools.sctrade.companion.domain.user.SettingRepository;
 import tools.sctrade.companion.exceptions.PublicationException;
 import tools.sctrade.companion.utils.AsynchronousProcessor;
+import tools.sctrade.companion.utils.LocalizationUtil;
 
 public class ScTradeToolsClient extends AsynchronousProcessor<CommoditySubmission>
     implements CommodityRepository, LocationRepository {
@@ -86,6 +87,9 @@ public void process(CommoditySubmission submission) {
           .toBodilessEntity();
       response.block();
       logger.info("Sent {} commodity listings to SC Trade Tools", submission.getListings().size());
+      notificationService.info(String.format(Locale.ROOT,
+          LocalizationUtil.get("infoCommodityListingsScTradeToolsOutput"),
+          submission.getListings().size()));
     } catch (WebClientResponseException e) {
       throw new PublicationException(e.getResponseBodyAsString());
     }

src/main/java/tools/sctrade/companion/utils/ImageUtil.java

@@ -25,6 +25,7 @@
 import org.opencv.core.Mat;
 import org.opencv.core.MatOfByte;
 import org.opencv.core.MatOfPoint;
+import org.opencv.core.Size;
 import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.imgproc.Imgproc;
 import org.slf4j.Logger;
@@ -179,9 +180,11 @@ public static BufferedImage applyOtsuBinarization(BufferedImage image) {
 
     try {
       Mat original = toMat(image);
+      Mat filtered = new Mat(original.rows(), original.cols(), original.type());
       Mat processed = new Mat(original.rows(), original.cols(), original.type());
 
-      Imgproc.threshold(original, processed, Imgproc.THRESH_BINARY, 255, Imgproc.THRESH_OTSU);
+      Imgproc.GaussianBlur(original, filtered, new Size(5, 5), 0);
+      Imgproc.threshold(filtered, processed, Imgproc.THRESH_BINARY, 255, Imgproc.THRESH_OTSU);
 
       return toBufferedImage(processed);
     } catch (IOException e) {

src/main/resources/bundles/localization.properties

@@ -17,11 +17,13 @@ tableColumnType = Type
 tableColumnMessage = Message
 infoProcessingScreenshot = Processing screenshot...
 infoCommodityListingsCsvOutput = Wrote %d commodity listings to '%s'
+infoCommodityListingsScTradeToolsOutput = Sent %d commodity listings to SC Trade Tools
 infoCommodityListingsRead = Read commodity listings
 warnNoLocation = Select the current shop/city name in the "Your inventories" dropdown
 errorNotEnoughColumns = [Code 1] Could not make out %d or more column(s) of text in:%n%s
 errorRectangleOutOfBounds = [Code 2] Rectangle '%s' is not within '%s'
 errorLocationNotFound = [Code 3] Could not extract location from: %s
 errorNoCloseString = [Code 4] Could not find string resembling '%s'
-errorNoListings = [Code 5] No listings were found in the image
-errorNoLocation = [Code 6] The shop/city name was never selected in the "Your inventories" dropdown
+errorNoListings = [Code 5] Listings could not be read
+errorNoLocation = [Code 6] The shop/city name was never selected in the "Your inventories" dropdown
+errorUnreadableTransactionType [Code 7] Could not find the buy & sell buttons