Changeset 111 for java-automator/src

Timestamp:

Apr 15, 2010, 12:19:42 PM (15 years ago)

Author:

Rick van der Zwet

Message:

Hacking with overlays and detections

File:

• java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java (modified) (11 diffs)

java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java

@@ -2 +2 @@
  * Rick van der Zwet <info@rickvanderzwet.nl>
  * License BSDLike http://rickvanderzwet.nl/LICENSE
+ * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written.
  */
 
-/* Based after http://ironbark.bendigo.latrobe.edu.au/~mary/DS/lectures/tutorials/ACSWorkshop98.html */
 package nl.rickvanderzwet.farmville;
 
@@ -12 +12 @@
 import java.awt.image.RasterFormatException;
 
-
-public class ScreenRecognition extends Frame implements ActionListener  {
+import javax.swing.BorderFactory;
+import javax.swing.JButton;
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JLayeredPane;
+import javax.swing.JPanel;
+import javax.swing.Timer;
+
+//class ScreenPainter extends Panel {
+//      private static final long serialVersionUID = -3073797542841857452L;
+//      BufferedImage foo = null; 
+//
+//      public void paint(Graphics g) {
+//              //      System.out.println("FOO");
+//              //      g.drawImage(foo,20, 50, this);
+//      }
+//}
+
+public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener   {
         /**
          * 
@@ -24 +41 @@
         static BufferedImage bi;
         static Robot bot;
-        
+
         static boolean freezeImage = false;
-        static boolean mouseClickEnabled = true;
+        static boolean mouseClickEnabled = false;
+
+        static boolean foundPlowLocation = false;
+
+        static Button redButton;
+        static Button greenButton;
+
+        Point boxLeftTop, boxRightBottom = new Point (0,0);
+        Boolean drawBox = false;
+        static Boolean draggingBox = false;
         
         private static void pressMouse() throws InterruptedException {
@@ -38 +64 @@
         public ScreenRecognition() {
                 setTitle("Screen Recognition Example");
+                JLayeredPane layeredPane = new JLayeredPane();
+
+                //layeredPane.setBorder(BorderFactory.createTitledBorder("Foo"));
+                //Panel foo = new ScreenPainter();
+                //foo.setBackground(Color.RED);
+            //foo.setBounds(70, 70, 50, 50);
+
+                JLabel bar = new JLabel();
+                bar.setText("FooBAR");
+                bar.setBounds(0, 0, 200, 20);
+            // Create 3 buttons
+            JPanel top = new JPanel();
+            top.setBackground(Color.white);
+            top.setBounds(20, 20, 50, 50);
+            JPanel middle = new JPanel();
+            middle.setBackground(Color.gray);
+            middle.setBounds(40, 40, 50, 50);
+            JPanel bottom = new JPanel();
+            bottom.setBackground(Color.black);
+            bottom.setBounds(60, 60, 50, 50);
+            
+            // Place the buttons in different layers
+//          layeredPane.add(foo, new Integer(5));
+//          layeredPane.add(bar, new Integer(5));
+//          layeredPane.add(middle,new Integer(0));
+//          layeredPane.add(top, new Integer(4));
+//          layeredPane.add(bottom, new Integer(0));
                 
                 Panel buttons = new Panel();
-                Button redButton = new Button("Red");
+                redButton = new Button("Red");
                 Button blueButton = new Button("Blue");
-                Button greenButton = new Button("Green");
+                greenButton = new Button("Green");
                 redButton.setName("redButton");
                 redButton.setLabel("Freeze Screen");
@@ -52 +105 @@
                 buttons.add(blueButton);
                 buttons.add(greenButton);
+                
+
+//          add(layeredPane);
                 add(buttons,"South");
 
-                 
+                        
                 setSize(400,300);
                 addWindowListener(new WindowAdapter() {
@@ -62 +118 @@
                 });
                 setVisible(true);
+            addMouseMotionListener( this );
+            addMouseListener( this );
+
         }
 
         public void actionPerformed(ActionEvent event) {
+                System.out.println("ActionPerformed");
                 Component c = (Component)event.getSource();
                 if (c.getName() == "redButton") {
-                        c.getToolkit().beep();
-                        try {
-                                Thread.sleep(3000);
-                        } catch (InterruptedException e) {
-                                // TODO Auto-generated catch block
-                                e.printStackTrace();
+                        if (freezeImage) {
+                                freezeImage = false;
+                                redButton.setLabel("Freeze image");
+                        } else {
+                                c.getToolkit().beep();
+
+                                /* Freeze screen after number of seconds seconds */
+                                ActionListener freezeScreen = new ActionListener() {
+                                        public void actionPerformed(ActionEvent evt) {
+                                                System.out.println(freezeImage);
+                                                freezeImage = true;
+                                                redButton.setLabel("Release image");
+                                        }
+                                };
+                                Timer foo = new Timer(3000, freezeScreen);
+                                foo.setRepeats(false);
+                                foo.start();
+                                redButton.setLabel("Freezing image...");
                         }
-                        freezeImage = !freezeImage;
                 } else if (c.getName() == "blueButton") {
                         mouseClickEnabled =  !mouseClickEnabled;
@@ -84 +155 @@
                 }
         }
-        
+
         public void paint(Graphics g) {
+                super.paint(g);
                 g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40);
                 g.drawString("PixelRGB:" + pixel.toString() ,150,40);
@@ -92 +164 @@
                 g.setColor(pixel);
                 g.fillRect(300,100,20,20);
+                
                 g.drawImage(bi,20, 50, this);
+
+                if (drawBox){
+                        g.setColor(Color.RED);
+                        g.drawRect(boxLeftTop.x, boxLeftTop.y, Math.abs(boxRightBottom.x - boxLeftTop.x), Math.abs(boxRightBottom.y - boxLeftTop.y));
+                }
+                
+                greenButton.setEnabled(foundPlowLocation);
+                
+                bi.createGraphics();
+                
         }
 
@@ -100 +183 @@
                 ScreenRecognition foo = new ScreenRecognition();
 
+                Point prevLocation = new Point(0,0);            
                 while (true) {
                         PointerInfo pointer = MouseInfo.getPointerInfo();
@@ -105 +189 @@
                         pixel = bot.getPixelColor(mouse.x, mouse.y);
                         try {
-                                if (!freezeImage) {
-                                bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
-                                                mouse.y + mouseSurround));
-                                bi = bi.getSubimage(mouse.x - mouseSurround, 
-                                                mouse.y - mouseSurround, 
-                                                mouseSurround * 2, mouseSurround * 2);
-                                
-                                /* See is enabled */
-                                Color target = new Color(255, 170, 0);
-                                int pixelTreshold = 10;
-                                
-                                search: for (int x = 0; x < bi.getWidth(); x++) {
-                                        for (int y = 0; y < bi.getHeight(); y++){
-                                                if (bi.getRGB(x, y) == target.getRGB()) {
-                                                        pixelTreshold--;
-                                                        if (pixelTreshold == 0) {
-                                                                break search;
+                                if (!freezeImage && !mouse.equals(prevLocation)) {
+                                        System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")");
+                                        prevLocation = mouse;
+                                        bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
+                                                        mouse.y + mouseSurround));
+                                        bi = bi.getSubimage(mouse.x - mouseSurround, 
+                                                        mouse.y - mouseSurround, 
+                                                        mouseSurround * 2, mouseSurround * 2);
+
+                                        /* See is enabled */
+                                        final Color target = new Color(255, 170, 0);
+                                        int pixelTreshold = 10;
+
+                                        search: for (int x = 0; x < bi.getWidth(); x++) {
+                                                for (int y = 0; y < bi.getHeight(); y++){
+                                                        if (bi.getRGB(x, y) == target.getRGB()) {
+                                                                pixelTreshold--;
+                                                                if (pixelTreshold == 0) {
+                                                                        break search;
+                                                                }
                                                         }
                                                 }
                                         }
-                                }
-                                if (pixelTreshold == 0) {
-                                        pressMouse();
-                                        System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
-                                }
+
+                                        /* Select text only */
+                                        final Color white = new Color(255, 255, 255);
+                                        final Color black = new Color(0, 0, 0);
+                                        for (int x = 0; x < bi.getWidth(); x++) {
+                                                for (int y = 0; y < bi.getHeight(); y++){
+                                                        if (bi.getRGB(x, y) != white.getRGB()) {
+                                                                bi.setRGB(x,y, black.getRGB());
+                                                        }
+                                                }
+                                        }
+                                                                                
+                                        /* Determine text borders */
+                                        Point leftTop = new Point(bi.getWidth(),bi.getHeight());
+                                        Point rightBottom = new Point(0,0);
+                                        for (int x = 0; x < bi.getWidth(); x++) {
+                                                for (int y = 0; y < bi.getHeight(); y++){
+                                                        if (bi.getRGB(x, y) == white.getRGB()) {
+                                                                if (x < leftTop.x && y < leftTop.y) {
+                                                                        leftTop.x = x; 
+                                                                        leftTop.y = y;
+                                                                }
+                                                                if (x > rightBottom.x && y > rightBottom.y) {
+                                                                        rightBottom.x = x;
+                                                                        rightBottom.y = y;
+                                                                }
+                                                        }
+                                                }
+                                        }
+
+                                        /* Do basic pattern recognition */
+                                        
+
+                                        if (pixelTreshold == 0) {
+                                                pressMouse();
+                                                System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
+                                        }
 
                                 }
@@ -143 +262 @@
 
         }
+
+
+        @Override
+        public void mouseDragged(MouseEvent e) {
+                boxRightBottom = e.getPoint();
+                //System.out.println("Dragged");
+
+        }
+
+        @Override
+        public void mouseMoved(MouseEvent e) {
+                //System.out.println("Moved");
+
+                // TODO Auto-generated method stub
+                
+        }
+        
+        public void mouseClicked (MouseEvent e) {
+                //System.out.println("Clicked");
+        }
+        
+        public void mouseReleased (MouseEvent e) {
+                //System.out.println("Released");
+                draggingBox = false;
+        }
+
+        @Override
+        public void mouseEntered(MouseEvent e) {
+                //System.out.println("Entered");
+
+                // TODO Auto-generated method stub
+                
+        }
+
+        @Override
+        public void mouseExited(MouseEvent e) {
+                //System.out.println("Exited");
+
+                // TODO Auto-generated method stub
+                
+        }
+
+        @Override
+        public void mousePressed(MouseEvent e) {
+                //System.out.println("Pressed");
+                drawBox = true;
+                boxLeftTop = e.getPoint();
+                draggingBox = true;
+                // TODO Auto-generated method stub
+                
+        }
 }