/* * Rick van der Zwet * License BSDLike http://rickvanderzwet.nl/LICENSE * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written. * XXX: Dragging sucks needs to be be around */ package nl.rickvanderzwet.farmville; import java.awt.*; import java.awt.event.*; import java.awt.image.BufferedImage; import java.awt.image.IndexColorModel; import java.awt.image.RasterFormatException; import javax.swing.JFrame; import javax.swing.JLabel; import javax.swing.JLayeredPane; import javax.swing.JPanel; import javax.swing.Timer; //class ScreenPainter extends Panel { // private static final long serialVersionUID = -3073797542841857452L; // BufferedImage foo = null; // // public void paint(Graphics g) { // // System.out.println("FOO"); // // g.drawImage(foo,20, 50, this); // } //} public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener { /** * */ private static final long serialVersionUID = -6742715532913330783L; static int mouseSurround = 120; Image image; static Color pixel; static Point mouse; static BufferedImage bi; static BufferedImage rawPattern; static BufferedImage pattern; static Robot bot; static boolean freezeImage = false; static boolean mouseClickEnabled = false; static boolean foundPlowLocation = false; static Button redButton; static Button greenButton; static Button sampleShotButton; static Point boxLeftTop = new Point (0,0); static Point boxRightBottom = new Point (0,0); Boolean drawBox = false; static Boolean draggingBox = false; private static void pressMouse() throws InterruptedException { if (mouseClickEnabled) { bot.mousePress(InputEvent.BUTTON1_MASK); bot.mouseRelease(InputEvent.BUTTON1_MASK); } } public ScreenRecognition() { setTitle("Screen Recognition Example"); JLayeredPane layeredPane = new JLayeredPane(); //layeredPane.setBorder(BorderFactory.createTitledBorder("Foo")); //Panel foo = new ScreenPainter(); //foo.setBackground(Color.RED); //foo.setBounds(70, 70, 50, 50); JLabel bar = new JLabel(); bar.setText("FooBAR"); bar.setBounds(0, 0, 200, 20); // Create 3 buttons JPanel top = new JPanel(); top.setBackground(Color.white); top.setBounds(20, 20, 50, 50); JPanel middle = new JPanel(); middle.setBackground(Color.gray); middle.setBounds(40, 40, 50, 50); JPanel bottom = new JPanel(); bottom.setBackground(Color.black); bottom.setBounds(60, 60, 50, 50); // Place the buttons in different layers // layeredPane.add(foo, new Integer(5)); // layeredPane.add(bar, new Integer(5)); // layeredPane.add(middle,new Integer(0)); // layeredPane.add(top, new Integer(4)); // layeredPane.add(bottom, new Integer(0)); Panel buttons = new Panel(); redButton = new Button("Red"); Button blueButton = new Button("Blue"); greenButton = new Button("Green"); redButton.setName("redButton"); redButton.setLabel("Freeze Screen"); redButton.addActionListener(this); blueButton.setName("blueButton"); blueButton.setLabel("Stop clicking"); blueButton.addActionListener(this); sampleShotButton = new Button("sampleShot"); sampleShotButton.setName("sampleShot"); sampleShotButton.setLabel("Sample shot"); sampleShotButton.addActionListener(this); buttons.add(redButton); buttons.add(blueButton); buttons.add(greenButton); buttons.add(sampleShotButton); // add(layeredPane); add(buttons,"South"); setSize(600,300); addWindowListener(new WindowAdapter() { public void windowClosing(WindowEvent e){ System.exit(0); } }); setVisible(true); addMouseMotionListener( this ); addMouseListener( this ); } public void actionPerformed(ActionEvent event) { Component c = (Component)event.getSource(); System.out.println("ActionPerformed: " + c.getName()); if (c.getName() == "redButton") { if (freezeImage) { freezeImage = false; redButton.setLabel("Freeze image"); } else { c.getToolkit().beep(); /* Freeze screen after number of seconds seconds */ ActionListener freezeScreen = new ActionListener() { public void actionPerformed(ActionEvent evt) { System.out.println(freezeImage); freezeImage = true; redButton.setLabel("Release image"); } }; Timer foo = new Timer(500, freezeScreen); foo.setRepeats(false); foo.start(); redButton.setLabel("Freezing image..."); } } else if (c.getName() == "blueButton") { mouseClickEnabled = !mouseClickEnabled; if (mouseClickEnabled) { ((Button) c).setLabel("Stop clicking"); } else { ((Button) c).setLabel("Start clicking"); } } else if (c.getName() == "sampleShot") { System.out.println("Point to use: " + boxLeftTop + boxRightBottom); /* Make sure we can select both ways */ Integer xStart = (boxLeftTop.x < boxRightBottom.x) ? boxLeftTop.x : boxRightBottom.x; Integer yStart = (boxLeftTop.y < boxRightBottom.y) ? boxLeftTop.y : boxRightBottom.y; Integer xWidth = Math.abs(boxRightBottom.x - boxLeftTop.x); Integer yWidth = Math.abs(boxRightBottom.y - boxLeftTop.y); //pattern = bi.getSubimage(xStart - 20 , yStart - 50, xWidth, yWidth); rawPattern = new BufferedImage(bi.getWidth(), bi.getHeight(), BufferedImage.TYPE_INT_RGB); bi.copyData(rawPattern.getRaster()); xStart -= 20; yStart -= 50; System.out.println("Values used: " + xStart + " ," + yStart + ", " + xWidth + "," +yWidth); pattern = rawPattern.getSubimage(xStart , yStart, xWidth, yWidth); } } public void paint(Graphics g) { super.paint(g); g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40); g.drawString("PixelRGB:" + pixel.toString() ,150,40); g.drawRect(18, 48, mouseSurround * 2 + 4, mouseSurround * 2 + 4); g.setColor(pixel); g.fillRect(300,100,20,20); g.drawImage(bi,20, 50, this); g.drawImage(pattern, 320, 50, this); if (drawBox){ g.setColor(Color.RED); /* Make sure we can select both ways */ Integer xStart = (boxLeftTop.x < boxRightBottom.x) ? boxLeftTop.x : boxRightBottom.x; Integer yStart = (boxLeftTop.y < boxRightBottom.y) ? boxLeftTop.y : boxRightBottom.y; Integer xWidth = Math.abs(boxRightBottom.x - boxLeftTop.x); Integer yWidth = Math.abs(boxRightBottom.y - boxLeftTop.y); g.drawRect(xStart, yStart, xWidth, yWidth); } greenButton.setEnabled(foundPlowLocation); bi.createGraphics(); } public static void main(String[] args) throws AWTException, InterruptedException { //bot.mouseMove(100, 100); bot = new Robot(); ScreenRecognition foo = new ScreenRecognition(); Point prevLocation = new Point(0,0); while (true) { PointerInfo pointer = MouseInfo.getPointerInfo(); mouse = pointer.getLocation(); pixel = bot.getPixelColor(mouse.x, mouse.y); try { if (!freezeImage && !mouse.equals(prevLocation)) { System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")"); prevLocation = mouse; /* XXX: Make sure we do not cross the boundaries */ bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround, mouse.y + mouseSurround)); Integer xBase = ((mouse.x - mouseSurround) < 0) ? mouseSurround : (mouse.x - mouseSurround); Integer yBase = ((mouse.y - mouseSurround) < 0) ? mouseSurround : (mouse.y - mouseSurround); bi = bi.getSubimage(xBase, yBase, mouseSurround * 2, mouseSurround * 2); /* See is enabled */ final Color target = new Color(255, 170, 0); int pixelTreshold = 10; search: for (int x = 0; x < bi.getWidth(); x++) { for (int y = 0; y < bi.getHeight(); y++){ if (bi.getRGB(x, y) == target.getRGB()) { pixelTreshold--; if (pixelTreshold == 0) { break search; } } } } /* Select text only */ final Color white = new Color(255, 255, 255); final Color black = new Color(0, 0, 0); for (int x = 0; x < bi.getWidth(); x++) { for (int y = 0; y < bi.getHeight(); y++){ if (bi.getRGB(x, y) != white.getRGB()) { bi.setRGB(x,y, black.getRGB()); } } } /* Determine text borders */ Point leftTop = new Point(bi.getWidth(),bi.getHeight()); Point rightBottom = new Point(0,0); for (int x = 0; x < bi.getWidth(); x++) { for (int y = 0; y < bi.getHeight(); y++){ if (bi.getRGB(x, y) == white.getRGB()) { if (x < leftTop.x && y < leftTop.y) { leftTop.x = x; leftTop.y = y; } if (x > rightBottom.x && y > rightBottom.y) { rightBottom.x = x; rightBottom.y = y; } } } } Boolean patternMatch = false; Point patLoc = new Point(0,0); if (pattern != null) { /* Do basic pattern recognition * XXX: Does not scale, does not support multiple objects * XXX: Maybe colour will help us over here as well */ nextx: for (int x = 0; x < bi.getWidth() - pattern.getWidth(); x++) { nexty: for (int y = 0; y < bi.getHeight() - pattern.getHeight(); y++) { for (int i = 0; i < pattern.getWidth(); i++) { for (int j = 0; j < pattern.getHeight(); j++) { if (bi.getRGB(x + i, y + j) != pattern.getRGB(i, j)) { //System.out.println("Fail at (" + x + "," + y + ") [" + // patLoc.x + "," + patLoc.y + "] " + // bi.getRGB(x + i, y + j) + "/" + pattern.getRGB(i, j)); continue nexty; } } } /* We are successful */ patternMatch = true; break nextx; } } } if(patternMatch) { System.out.println("Match found at location: " + patLoc); } if (pixelTreshold == 0) { pressMouse(); System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]"); } } foo.repaint(); } catch (IllegalArgumentException e) { continue; } catch (RasterFormatException e) { continue; } // Allow time to focus Firefox window Thread.sleep(100); } } @Override public void mouseDragged(MouseEvent e) { boxRightBottom = e.getPoint(); //System.out.println("Dragged"); } @Override public void mouseMoved(MouseEvent e) { //System.out.println("Moved"); // TODO Auto-generated method stub } public void mouseClicked (MouseEvent e) { //System.out.println("Clicked"); } public void mouseReleased (MouseEvent e) { //System.out.println("Released"); draggingBox = false; } @Override public void mouseEntered(MouseEvent e) { //System.out.println("Entered"); // TODO Auto-generated method stub } @Override public void mouseExited(MouseEvent e) { //System.out.println("Exited"); // TODO Auto-generated method stub } @Override public void mousePressed(MouseEvent e) { //System.out.println("Pressed"); drawBox = true; boxLeftTop = e.getPoint(); draggingBox = true; // TODO Auto-generated method stub } }