source: java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java@ 405

Last change on this file since 405 was 112, checked in by Rick van der Zwet, 15 years ago

Basic object detection working, next object workflow and action list based uppon that.
File size: 11.2 KB
RevLine 
[103]1/*
2 * Rick van der Zwet <info@rickvanderzwet.nl>
3 * License BSDLike http://rickvanderzwet.nl/LICENSE
[111]4 * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written.
[112]5 * XXX: Dragging sucks needs to be be around
[103]6 */
7
8package nl.rickvanderzwet.farmville;
9
10import java.awt.*;
11import java.awt.event.*;
12import java.awt.image.BufferedImage;
[112]13import java.awt.image.IndexColorModel;
[103]14import java.awt.image.RasterFormatException;
15
[111]16import javax.swing.JFrame;
17import javax.swing.JLabel;
18import javax.swing.JLayeredPane;
19import javax.swing.JPanel;
20import javax.swing.Timer;
[103]21
[111]22//class ScreenPainter extends Panel {
23// private static final long serialVersionUID = -3073797542841857452L;
24// BufferedImage foo = null;
25//
26// public void paint(Graphics g) {
27// // System.out.println("FOO");
28// // g.drawImage(foo,20, 50, this);
29// }
30//}
31
32public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener {
[108]33 /**
34 *
35 */
36 private static final long serialVersionUID = -6742715532913330783L;
[103]37 static int mouseSurround = 120;
38 Image image;
39 static Color pixel;
40 static Point mouse;
41 static BufferedImage bi;
[112]42 static BufferedImage rawPattern;
43 static BufferedImage pattern;
[104]44 static Robot bot;
[111]45
[104]46 static boolean freezeImage = false;
[111]47 static boolean mouseClickEnabled = false;
48
49 static boolean foundPlowLocation = false;
50
51 static Button redButton;
52 static Button greenButton;
[112]53 static Button sampleShotButton;
[111]54
[112]55
56 static Point boxLeftTop = new Point (0,0);
57 static Point boxRightBottom = new Point (0,0);
[111]58 Boolean drawBox = false;
59 static Boolean draggingBox = false;
[104]60
61 private static void pressMouse() throws InterruptedException {
[108]62 if (mouseClickEnabled) {
63 bot.mousePress(InputEvent.BUTTON1_MASK);
64 bot.mouseRelease(InputEvent.BUTTON1_MASK);
65 }
[104]66
67 }
68
[103]69 public ScreenRecognition() {
[108]70 setTitle("Screen Recognition Example");
[111]71 JLayeredPane layeredPane = new JLayeredPane();
72
73 //layeredPane.setBorder(BorderFactory.createTitledBorder("Foo"));
74 //Panel foo = new ScreenPainter();
75 //foo.setBackground(Color.RED);
76 //foo.setBounds(70, 70, 50, 50);
77
78 JLabel bar = new JLabel();
79 bar.setText("FooBAR");
80 bar.setBounds(0, 0, 200, 20);
81 // Create 3 buttons
82 JPanel top = new JPanel();
83 top.setBackground(Color.white);
84 top.setBounds(20, 20, 50, 50);
85 JPanel middle = new JPanel();
86 middle.setBackground(Color.gray);
87 middle.setBounds(40, 40, 50, 50);
88 JPanel bottom = new JPanel();
89 bottom.setBackground(Color.black);
90 bottom.setBounds(60, 60, 50, 50);
91
92 // Place the buttons in different layers
93// layeredPane.add(foo, new Integer(5));
94// layeredPane.add(bar, new Integer(5));
95// layeredPane.add(middle,new Integer(0));
96// layeredPane.add(top, new Integer(4));
97// layeredPane.add(bottom, new Integer(0));
[103]98
99 Panel buttons = new Panel();
[111]100 redButton = new Button("Red");
[103]101 Button blueButton = new Button("Blue");
[111]102 greenButton = new Button("Green");
[103]103 redButton.setName("redButton");
[108]104 redButton.setLabel("Freeze Screen");
[103]105 redButton.addActionListener(this);
106 blueButton.setName("blueButton");
[108]107 blueButton.setLabel("Stop clicking");
[103]108 blueButton.addActionListener(this);
[112]109
110 sampleShotButton = new Button("sampleShot");
111 sampleShotButton.setName("sampleShot");
112 sampleShotButton.setLabel("Sample shot");
113 sampleShotButton.addActionListener(this);
[103]114 buttons.add(redButton);
115 buttons.add(blueButton);
116 buttons.add(greenButton);
[112]117 buttons.add(sampleShotButton);
[111]118
119
120// add(layeredPane);
[103]121 add(buttons,"South");
122
[111]123
[112]124 setSize(600,300);
[103]125 addWindowListener(new WindowAdapter() {
126 public void windowClosing(WindowEvent e){
127 System.exit(0);
128 }
129 });
130 setVisible(true);
[111]131 addMouseMotionListener( this );
132 addMouseListener( this );
133
[103]134 }
135
136 public void actionPerformed(ActionEvent event) {
137 Component c = (Component)event.getSource();
[112]138 System.out.println("ActionPerformed: " + c.getName());
[104]139 if (c.getName() == "redButton") {
[111]140 if (freezeImage) {
141 freezeImage = false;
142 redButton.setLabel("Freeze image");
143 } else {
144 c.getToolkit().beep();
145
146 /* Freeze screen after number of seconds seconds */
147 ActionListener freezeScreen = new ActionListener() {
148 public void actionPerformed(ActionEvent evt) {
149 System.out.println(freezeImage);
150 freezeImage = true;
151 redButton.setLabel("Release image");
152 }
153 };
[112]154 Timer foo = new Timer(500, freezeScreen);
[111]155 foo.setRepeats(false);
156 foo.start();
157 redButton.setLabel("Freezing image...");
[104]158 }
[108]159 } else if (c.getName() == "blueButton") {
160 mouseClickEnabled = !mouseClickEnabled;
161 if (mouseClickEnabled) {
162 ((Button) c).setLabel("Stop clicking");
163 } else {
164 ((Button) c).setLabel("Start clicking");
165 }
[112]166 } else if (c.getName() == "sampleShot") {
167 System.out.println("Point to use: " + boxLeftTop + boxRightBottom);
168
169 /* Make sure we can select both ways */
170 Integer xStart = (boxLeftTop.x < boxRightBottom.x) ? boxLeftTop.x : boxRightBottom.x;
171 Integer yStart = (boxLeftTop.y < boxRightBottom.y) ? boxLeftTop.y : boxRightBottom.y;
172
173 Integer xWidth = Math.abs(boxRightBottom.x - boxLeftTop.x);
174 Integer yWidth = Math.abs(boxRightBottom.y - boxLeftTop.y);
175 //pattern = bi.getSubimage(xStart - 20 , yStart - 50, xWidth, yWidth);
176 rawPattern = new BufferedImage(bi.getWidth(), bi.getHeight(), BufferedImage.TYPE_INT_RGB);
177 bi.copyData(rawPattern.getRaster());
178 xStart -= 20;
179 yStart -= 50;
180 System.out.println("Values used: " + xStart + " ," + yStart + ", " + xWidth + "," +yWidth);
181 pattern = rawPattern.getSubimage(xStart , yStart, xWidth, yWidth);
[104]182 }
[103]183 }
[112]184
[103]185 public void paint(Graphics g) {
[111]186 super.paint(g);
[103]187 g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40);
188 g.drawString("PixelRGB:" + pixel.toString() ,150,40);
189
190 g.drawRect(18, 48, mouseSurround * 2 + 4, mouseSurround * 2 + 4);
[104]191 g.setColor(pixel);
192 g.fillRect(300,100,20,20);
[111]193
[103]194 g.drawImage(bi,20, 50, this);
[112]195 g.drawImage(pattern, 320, 50, this);
[111]196
197 if (drawBox){
198 g.setColor(Color.RED);
[112]199
200 /* Make sure we can select both ways */
201 Integer xStart = (boxLeftTop.x < boxRightBottom.x) ? boxLeftTop.x : boxRightBottom.x;
202 Integer yStart = (boxLeftTop.y < boxRightBottom.y) ? boxLeftTop.y : boxRightBottom.y;
203
204 Integer xWidth = Math.abs(boxRightBottom.x - boxLeftTop.x);
205 Integer yWidth = Math.abs(boxRightBottom.y - boxLeftTop.y);
206 g.drawRect(xStart, yStart, xWidth, yWidth);
[111]207 }
208
209 greenButton.setEnabled(foundPlowLocation);
210 bi.createGraphics();
211
[103]212 }
213
214 public static void main(String[] args) throws AWTException, InterruptedException {
215 //bot.mouseMove(100, 100);
[104]216 bot = new Robot();
[103]217 ScreenRecognition foo = new ScreenRecognition();
218
[111]219 Point prevLocation = new Point(0,0);
[103]220 while (true) {
221 PointerInfo pointer = MouseInfo.getPointerInfo();
222 mouse = pointer.getLocation();
223 pixel = bot.getPixelColor(mouse.x, mouse.y);
224 try {
[111]225 if (!freezeImage && !mouse.equals(prevLocation)) {
226 System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")");
227 prevLocation = mouse;
[112]228
229 /* XXX: Make sure we do not cross the boundaries */
230
[111]231 bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
232 mouse.y + mouseSurround));
233
[112]234 Integer xBase = ((mouse.x - mouseSurround) < 0) ? mouseSurround : (mouse.x - mouseSurround);
235 Integer yBase = ((mouse.y - mouseSurround) < 0) ? mouseSurround : (mouse.y - mouseSurround);
236 bi = bi.getSubimage(xBase, yBase, mouseSurround * 2, mouseSurround * 2);
237
[111]238 /* See is enabled */
239 final Color target = new Color(255, 170, 0);
240 int pixelTreshold = 10;
241
242 search: for (int x = 0; x < bi.getWidth(); x++) {
243 for (int y = 0; y < bi.getHeight(); y++){
244 if (bi.getRGB(x, y) == target.getRGB()) {
245 pixelTreshold--;
246 if (pixelTreshold == 0) {
247 break search;
248 }
[104]249 }
250 }
251 }
252
[111]253 /* Select text only */
254 final Color white = new Color(255, 255, 255);
255 final Color black = new Color(0, 0, 0);
256 for (int x = 0; x < bi.getWidth(); x++) {
257 for (int y = 0; y < bi.getHeight(); y++){
258 if (bi.getRGB(x, y) != white.getRGB()) {
259 bi.setRGB(x,y, black.getRGB());
260 }
261 }
262 }
263
264 /* Determine text borders */
265 Point leftTop = new Point(bi.getWidth(),bi.getHeight());
266 Point rightBottom = new Point(0,0);
267 for (int x = 0; x < bi.getWidth(); x++) {
268 for (int y = 0; y < bi.getHeight(); y++){
269 if (bi.getRGB(x, y) == white.getRGB()) {
270 if (x < leftTop.x && y < leftTop.y) {
271 leftTop.x = x;
272 leftTop.y = y;
273 }
274 if (x > rightBottom.x && y > rightBottom.y) {
275 rightBottom.x = x;
276 rightBottom.y = y;
277 }
278 }
279 }
280 }
281
[112]282 Boolean patternMatch = false;
283 Point patLoc = new Point(0,0);
284
285 if (pattern != null) {
286 /* Do basic pattern recognition
287 * XXX: Does not scale, does not support multiple objects
288 * XXX: Maybe colour will help us over here as well
289 */
290 nextx: for (int x = 0; x < bi.getWidth() - pattern.getWidth(); x++) {
291 nexty: for (int y = 0; y < bi.getHeight() - pattern.getHeight(); y++) {
292 for (int i = 0; i < pattern.getWidth(); i++) {
293 for (int j = 0; j < pattern.getHeight(); j++) {
294 if (bi.getRGB(x + i, y + j) != pattern.getRGB(i, j)) {
295 //System.out.println("Fail at (" + x + "," + y + ") [" +
296 // patLoc.x + "," + patLoc.y + "] " +
297 // bi.getRGB(x + i, y + j) + "/" + pattern.getRGB(i, j));
298 continue nexty;
299 }
300 }
301 }
302 /* We are successful */
303 patternMatch = true;
304 break nextx;
305 }
306 }
307 }
[111]308
[112]309 if(patternMatch) {
310 System.out.println("Match found at location: " + patLoc);
311 }
312
[111]313 if (pixelTreshold == 0) {
314 pressMouse();
315 System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
316 }
317
[104]318 }
[103]319 foo.repaint();
320 } catch (IllegalArgumentException e) {
321 continue;
322 } catch (RasterFormatException e) {
323 continue;
324 }
325 // Allow time to focus Firefox window
326 Thread.sleep(100);
327 }
328
329 }
[111]330
331
332 @Override
333 public void mouseDragged(MouseEvent e) {
334 boxRightBottom = e.getPoint();
335 //System.out.println("Dragged");
336
337 }
338
339 @Override
340 public void mouseMoved(MouseEvent e) {
341 //System.out.println("Moved");
342
343 // TODO Auto-generated method stub
344
345 }
346
347 public void mouseClicked (MouseEvent e) {
348 //System.out.println("Clicked");
349 }
350
351 public void mouseReleased (MouseEvent e) {
352 //System.out.println("Released");
353 draggingBox = false;
354 }
355
356 @Override
357 public void mouseEntered(MouseEvent e) {
358 //System.out.println("Entered");
359
360 // TODO Auto-generated method stub
361
362 }
363
364 @Override
365 public void mouseExited(MouseEvent e) {
366 //System.out.println("Exited");
367
368 // TODO Auto-generated method stub
369
370 }
371
372 @Override
373 public void mousePressed(MouseEvent e) {
374 //System.out.println("Pressed");
375 drawBox = true;
376 boxLeftTop = e.getPoint();
377 draggingBox = true;
378 // TODO Auto-generated method stub
379
380 }
[103]381}
Note: See TracBrowser for help on using the repository browser.