Index: /java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java
===================================================================
--- /java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java	(revision 110)
+++ /java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java	(revision 111)
@@ -2,7 +2,7 @@
  * Rick van der Zwet <info@rickvanderzwet.nl>
  * License BSDLike http://rickvanderzwet.nl/LICENSE
+ * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written.
  */
 
-/* Based after http://ironbark.bendigo.latrobe.edu.au/~mary/DS/lectures/tutorials/ACSWorkshop98.html */
 package nl.rickvanderzwet.farmville;
 
@@ -12,6 +12,23 @@
 import java.awt.image.RasterFormatException;
 
-
-public class ScreenRecognition extends Frame implements ActionListener  {
+import javax.swing.BorderFactory;
+import javax.swing.JButton;
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JLayeredPane;
+import javax.swing.JPanel;
+import javax.swing.Timer;
+
+//class ScreenPainter extends Panel {
+//	private static final long serialVersionUID = -3073797542841857452L;
+//	BufferedImage foo = null; 
+//
+//	public void paint(Graphics g) {
+//		//	System.out.println("FOO");
+//		//	g.drawImage(foo,20, 50, this);
+//	}
+//}
+
+public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener   {
 	/**
 	 * 
@@ -24,7 +41,16 @@
 	static BufferedImage bi;
 	static Robot bot;
-	
+
 	static boolean freezeImage = false;
-	static boolean mouseClickEnabled = true;
+	static boolean mouseClickEnabled = false;
+
+	static boolean foundPlowLocation = false;
+
+	static Button redButton;
+	static Button greenButton;
+
+	Point boxLeftTop, boxRightBottom = new Point (0,0);
+	Boolean drawBox = false;
+	static Boolean draggingBox = false;
 	
 	private static void pressMouse() throws InterruptedException {
@@ -38,9 +64,36 @@
 	public ScreenRecognition() {
 		setTitle("Screen Recognition Example");
+		JLayeredPane layeredPane = new JLayeredPane();
+
+		//layeredPane.setBorder(BorderFactory.createTitledBorder("Foo"));
+		//Panel foo = new ScreenPainter();
+		//foo.setBackground(Color.RED);
+	    //foo.setBounds(70, 70, 50, 50);
+
+		JLabel bar = new JLabel();
+		bar.setText("FooBAR");
+		bar.setBounds(0, 0, 200, 20);
+	    // Create 3 buttons
+	    JPanel top = new JPanel();
+	    top.setBackground(Color.white);
+	    top.setBounds(20, 20, 50, 50);
+	    JPanel middle = new JPanel();
+	    middle.setBackground(Color.gray);
+	    middle.setBounds(40, 40, 50, 50);
+	    JPanel bottom = new JPanel();
+	    bottom.setBackground(Color.black);
+	    bottom.setBounds(60, 60, 50, 50);
+	    
+	    // Place the buttons in different layers
+//	    layeredPane.add(foo, new Integer(5));
+//	    layeredPane.add(bar, new Integer(5));
+//	    layeredPane.add(middle,new Integer(0));
+//	    layeredPane.add(top, new Integer(4));
+//	    layeredPane.add(bottom, new Integer(0));
 		
 		Panel buttons = new Panel();
-		Button redButton = new Button("Red");
+		redButton = new Button("Red");
 		Button blueButton = new Button("Blue");
-		Button greenButton = new Button("Green");
+		greenButton = new Button("Green");
 		redButton.setName("redButton");
 		redButton.setLabel("Freeze Screen");
@@ -52,7 +105,10 @@
 		buttons.add(blueButton);
 		buttons.add(greenButton);
+		
+
+//	    add(layeredPane);
 		add(buttons,"South");
 
-		 
+			
 		setSize(400,300);
 		addWindowListener(new WindowAdapter() {
@@ -62,17 +118,32 @@
 		});
 		setVisible(true);
+	    addMouseMotionListener( this );
+	    addMouseListener( this );
+
 	}
 
 	public void actionPerformed(ActionEvent event) {
+		System.out.println("ActionPerformed");
 		Component c = (Component)event.getSource();
 		if (c.getName() == "redButton") {
-			c.getToolkit().beep();
-			try {
-				Thread.sleep(3000);
-			} catch (InterruptedException e) {
-				// TODO Auto-generated catch block
-				e.printStackTrace();
+			if (freezeImage) {
+				freezeImage = false;
+				redButton.setLabel("Freeze image");
+			} else {
+				c.getToolkit().beep();
+
+				/* Freeze screen after number of seconds seconds */
+				ActionListener freezeScreen = new ActionListener() {
+					public void actionPerformed(ActionEvent evt) {
+						System.out.println(freezeImage);
+						freezeImage = true;
+						redButton.setLabel("Release image");
+					}
+				};
+				Timer foo = new Timer(3000, freezeScreen);
+				foo.setRepeats(false);
+				foo.start();
+				redButton.setLabel("Freezing image...");
 			}
-			freezeImage = !freezeImage;
 		} else if (c.getName() == "blueButton") {
 			mouseClickEnabled =  !mouseClickEnabled;
@@ -84,6 +155,7 @@
 		}
 	}
-	
+
 	public void paint(Graphics g) {
+		super.paint(g);
 		g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40);
 		g.drawString("PixelRGB:" + pixel.toString() ,150,40);
@@ -92,5 +164,16 @@
 		g.setColor(pixel);
 		g.fillRect(300,100,20,20);
+		
 		g.drawImage(bi,20, 50, this);
+
+		if (drawBox){
+			g.setColor(Color.RED);
+			g.drawRect(boxLeftTop.x, boxLeftTop.y, Math.abs(boxRightBottom.x - boxLeftTop.x), Math.abs(boxRightBottom.y - boxLeftTop.y));
+		}
+		
+		greenButton.setEnabled(foundPlowLocation);
+		
+		bi.createGraphics();
+		
 	}
 
@@ -100,4 +183,5 @@
 		ScreenRecognition foo = new ScreenRecognition();
 
+		Point prevLocation = new Point(0,0);		
 		while (true) {
 			PointerInfo pointer = MouseInfo.getPointerInfo();
@@ -105,29 +189,64 @@
 			pixel = bot.getPixelColor(mouse.x, mouse.y);
 			try {
-				if (!freezeImage) {
-				bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
-						mouse.y + mouseSurround));
-				bi = bi.getSubimage(mouse.x - mouseSurround, 
-						mouse.y - mouseSurround, 
-						mouseSurround * 2, mouseSurround * 2);
-				
-				/* See is enabled */
-				Color target = new Color(255, 170, 0);
-				int pixelTreshold = 10;
-				
-				search: for (int x = 0; x < bi.getWidth(); x++) {
-					for (int y = 0; y < bi.getHeight(); y++){
-						if (bi.getRGB(x, y) == target.getRGB()) {
-							pixelTreshold--;
-							if (pixelTreshold == 0) {
-								break search;
+				if (!freezeImage && !mouse.equals(prevLocation)) {
+					System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")");
+					prevLocation = mouse;
+					bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
+							mouse.y + mouseSurround));
+					bi = bi.getSubimage(mouse.x - mouseSurround, 
+							mouse.y - mouseSurround, 
+							mouseSurround * 2, mouseSurround * 2);
+
+					/* See is enabled */
+					final Color target = new Color(255, 170, 0);
+					int pixelTreshold = 10;
+
+					search: for (int x = 0; x < bi.getWidth(); x++) {
+						for (int y = 0; y < bi.getHeight(); y++){
+							if (bi.getRGB(x, y) == target.getRGB()) {
+								pixelTreshold--;
+								if (pixelTreshold == 0) {
+									break search;
+								}
 							}
 						}
 					}
-				}
-				if (pixelTreshold == 0) {
-					pressMouse();
-					System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
-				}
+
+					/* Select text only */
+					final Color white = new Color(255, 255, 255);
+					final Color black = new Color(0, 0, 0);
+					for (int x = 0; x < bi.getWidth(); x++) {
+						for (int y = 0; y < bi.getHeight(); y++){
+							if (bi.getRGB(x, y) != white.getRGB()) {
+								bi.setRGB(x,y, black.getRGB());
+							}
+						}
+					}
+										
+					/* Determine text borders */
+					Point leftTop = new Point(bi.getWidth(),bi.getHeight());
+					Point rightBottom = new Point(0,0);
+					for (int x = 0; x < bi.getWidth(); x++) {
+						for (int y = 0; y < bi.getHeight(); y++){
+							if (bi.getRGB(x, y) == white.getRGB()) {
+								if (x < leftTop.x && y < leftTop.y) {
+									leftTop.x = x; 
+									leftTop.y = y;
+								}
+								if (x > rightBottom.x && y > rightBottom.y) {
+									rightBottom.x = x;
+									rightBottom.y = y;
+								}
+							}
+						}
+					}
+
+					/* Do basic pattern recognition */
+					
+
+					if (pixelTreshold == 0) {
+						pressMouse();
+						System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
+					}
 
 				}
@@ -143,3 +262,54 @@
 
 	}
+
+
+	@Override
+	public void mouseDragged(MouseEvent e) {
+		boxRightBottom = e.getPoint();
+		//System.out.println("Dragged");
+
+	}
+
+	@Override
+	public void mouseMoved(MouseEvent e) {
+		//System.out.println("Moved");
+
+		// TODO Auto-generated method stub
+		
+	}
+	
+	public void mouseClicked (MouseEvent e) {
+		//System.out.println("Clicked");
+	}
+	
+	public void mouseReleased (MouseEvent e) {
+		//System.out.println("Released");
+		draggingBox = false;
+	}
+
+	@Override
+	public void mouseEntered(MouseEvent e) {
+		//System.out.println("Entered");
+
+		// TODO Auto-generated method stub
+		
+	}
+
+	@Override
+	public void mouseExited(MouseEvent e) {
+		//System.out.println("Exited");
+
+		// TODO Auto-generated method stub
+		
+	}
+
+	@Override
+	public void mousePressed(MouseEvent e) {
+		//System.out.println("Pressed");
+		drawBox = true;
+		boxLeftTop = e.getPoint();
+		draggingBox = true;
+		// TODO Auto-generated method stub
+		
+	}
 }
