source: java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java@ 111

Last change on this file since 111 was 111, checked in by Rick van der Zwet, 15 years ago

Hacking with overlays and detections
File size: 8.4 KB
RevLine 
[103]1/*
2 * Rick van der Zwet <info@rickvanderzwet.nl>
3 * License BSDLike http://rickvanderzwet.nl/LICENSE
[111]4 * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written.
[103]5 */
6
7package nl.rickvanderzwet.farmville;
8
9import java.awt.*;
10import java.awt.event.*;
11import java.awt.image.BufferedImage;
12import java.awt.image.RasterFormatException;
13
[111]14import javax.swing.BorderFactory;
15import javax.swing.JButton;
16import javax.swing.JFrame;
17import javax.swing.JLabel;
18import javax.swing.JLayeredPane;
19import javax.swing.JPanel;
20import javax.swing.Timer;
[103]21
[111]22//class ScreenPainter extends Panel {
23// private static final long serialVersionUID = -3073797542841857452L;
24// BufferedImage foo = null;
25//
26// public void paint(Graphics g) {
27// // System.out.println("FOO");
28// // g.drawImage(foo,20, 50, this);
29// }
30//}
31
32public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener {
[108]33 /**
34 *
35 */
36 private static final long serialVersionUID = -6742715532913330783L;
[103]37 static int mouseSurround = 120;
38 Image image;
39 static Color pixel;
40 static Point mouse;
41 static BufferedImage bi;
[104]42 static Robot bot;
[111]43
[104]44 static boolean freezeImage = false;
[111]45 static boolean mouseClickEnabled = false;
46
47 static boolean foundPlowLocation = false;
48
49 static Button redButton;
50 static Button greenButton;
51
52 Point boxLeftTop, boxRightBottom = new Point (0,0);
53 Boolean drawBox = false;
54 static Boolean draggingBox = false;
[104]55
56 private static void pressMouse() throws InterruptedException {
[108]57 if (mouseClickEnabled) {
58 bot.mousePress(InputEvent.BUTTON1_MASK);
59 bot.mouseRelease(InputEvent.BUTTON1_MASK);
60 }
[104]61
62 }
63
[103]64 public ScreenRecognition() {
[108]65 setTitle("Screen Recognition Example");
[111]66 JLayeredPane layeredPane = new JLayeredPane();
67
68 //layeredPane.setBorder(BorderFactory.createTitledBorder("Foo"));
69 //Panel foo = new ScreenPainter();
70 //foo.setBackground(Color.RED);
71 //foo.setBounds(70, 70, 50, 50);
72
73 JLabel bar = new JLabel();
74 bar.setText("FooBAR");
75 bar.setBounds(0, 0, 200, 20);
76 // Create 3 buttons
77 JPanel top = new JPanel();
78 top.setBackground(Color.white);
79 top.setBounds(20, 20, 50, 50);
80 JPanel middle = new JPanel();
81 middle.setBackground(Color.gray);
82 middle.setBounds(40, 40, 50, 50);
83 JPanel bottom = new JPanel();
84 bottom.setBackground(Color.black);
85 bottom.setBounds(60, 60, 50, 50);
86
87 // Place the buttons in different layers
88// layeredPane.add(foo, new Integer(5));
89// layeredPane.add(bar, new Integer(5));
90// layeredPane.add(middle,new Integer(0));
91// layeredPane.add(top, new Integer(4));
92// layeredPane.add(bottom, new Integer(0));
[103]93
94 Panel buttons = new Panel();
[111]95 redButton = new Button("Red");
[103]96 Button blueButton = new Button("Blue");
[111]97 greenButton = new Button("Green");
[103]98 redButton.setName("redButton");
[108]99 redButton.setLabel("Freeze Screen");
[103]100 redButton.addActionListener(this);
101 blueButton.setName("blueButton");
[108]102 blueButton.setLabel("Stop clicking");
[103]103 blueButton.addActionListener(this);
104 buttons.add(redButton);
105 buttons.add(blueButton);
106 buttons.add(greenButton);
[111]107
108
109// add(layeredPane);
[103]110 add(buttons,"South");
111
[111]112
[103]113 setSize(400,300);
114 addWindowListener(new WindowAdapter() {
115 public void windowClosing(WindowEvent e){
116 System.exit(0);
117 }
118 });
119 setVisible(true);
[111]120 addMouseMotionListener( this );
121 addMouseListener( this );
122
[103]123 }
124
125 public void actionPerformed(ActionEvent event) {
[111]126 System.out.println("ActionPerformed");
[103]127 Component c = (Component)event.getSource();
[104]128 if (c.getName() == "redButton") {
[111]129 if (freezeImage) {
130 freezeImage = false;
131 redButton.setLabel("Freeze image");
132 } else {
133 c.getToolkit().beep();
134
135 /* Freeze screen after number of seconds seconds */
136 ActionListener freezeScreen = new ActionListener() {
137 public void actionPerformed(ActionEvent evt) {
138 System.out.println(freezeImage);
139 freezeImage = true;
140 redButton.setLabel("Release image");
141 }
142 };
143 Timer foo = new Timer(3000, freezeScreen);
144 foo.setRepeats(false);
145 foo.start();
146 redButton.setLabel("Freezing image...");
[104]147 }
[108]148 } else if (c.getName() == "blueButton") {
149 mouseClickEnabled = !mouseClickEnabled;
150 if (mouseClickEnabled) {
151 ((Button) c).setLabel("Stop clicking");
152 } else {
153 ((Button) c).setLabel("Start clicking");
154 }
[104]155 }
[103]156 }
[111]157
[103]158 public void paint(Graphics g) {
[111]159 super.paint(g);
[103]160 g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40);
161 g.drawString("PixelRGB:" + pixel.toString() ,150,40);
162
163 g.drawRect(18, 48, mouseSurround * 2 + 4, mouseSurround * 2 + 4);
[104]164 g.setColor(pixel);
165 g.fillRect(300,100,20,20);
[111]166
[103]167 g.drawImage(bi,20, 50, this);
[111]168
169 if (drawBox){
170 g.setColor(Color.RED);
171 g.drawRect(boxLeftTop.x, boxLeftTop.y, Math.abs(boxRightBottom.x - boxLeftTop.x), Math.abs(boxRightBottom.y - boxLeftTop.y));
172 }
173
174 greenButton.setEnabled(foundPlowLocation);
175
176 bi.createGraphics();
177
[103]178 }
179
180 public static void main(String[] args) throws AWTException, InterruptedException {
181 //bot.mouseMove(100, 100);
[104]182 bot = new Robot();
[103]183 ScreenRecognition foo = new ScreenRecognition();
184
[111]185 Point prevLocation = new Point(0,0);
[103]186 while (true) {
187 PointerInfo pointer = MouseInfo.getPointerInfo();
188 mouse = pointer.getLocation();
189 pixel = bot.getPixelColor(mouse.x, mouse.y);
190 try {
[111]191 if (!freezeImage && !mouse.equals(prevLocation)) {
192 System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")");
193 prevLocation = mouse;
194 bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
195 mouse.y + mouseSurround));
196 bi = bi.getSubimage(mouse.x - mouseSurround,
197 mouse.y - mouseSurround,
198 mouseSurround * 2, mouseSurround * 2);
199
200 /* See is enabled */
201 final Color target = new Color(255, 170, 0);
202 int pixelTreshold = 10;
203
204 search: for (int x = 0; x < bi.getWidth(); x++) {
205 for (int y = 0; y < bi.getHeight(); y++){
206 if (bi.getRGB(x, y) == target.getRGB()) {
207 pixelTreshold--;
208 if (pixelTreshold == 0) {
209 break search;
210 }
[104]211 }
212 }
213 }
214
[111]215 /* Select text only */
216 final Color white = new Color(255, 255, 255);
217 final Color black = new Color(0, 0, 0);
218 for (int x = 0; x < bi.getWidth(); x++) {
219 for (int y = 0; y < bi.getHeight(); y++){
220 if (bi.getRGB(x, y) != white.getRGB()) {
221 bi.setRGB(x,y, black.getRGB());
222 }
223 }
224 }
225
226 /* Determine text borders */
227 Point leftTop = new Point(bi.getWidth(),bi.getHeight());
228 Point rightBottom = new Point(0,0);
229 for (int x = 0; x < bi.getWidth(); x++) {
230 for (int y = 0; y < bi.getHeight(); y++){
231 if (bi.getRGB(x, y) == white.getRGB()) {
232 if (x < leftTop.x && y < leftTop.y) {
233 leftTop.x = x;
234 leftTop.y = y;
235 }
236 if (x > rightBottom.x && y > rightBottom.y) {
237 rightBottom.x = x;
238 rightBottom.y = y;
239 }
240 }
241 }
242 }
243
244 /* Do basic pattern recognition */
245
246
247 if (pixelTreshold == 0) {
248 pressMouse();
249 System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
250 }
251
[104]252 }
[103]253 foo.repaint();
254 } catch (IllegalArgumentException e) {
255 continue;
256 } catch (RasterFormatException e) {
257 continue;
258 }
259 // Allow time to focus Firefox window
260 Thread.sleep(100);
261 }
262
263 }
[111]264
265
266 @Override
267 public void mouseDragged(MouseEvent e) {
268 boxRightBottom = e.getPoint();
269 //System.out.println("Dragged");
270
271 }
272
273 @Override
274 public void mouseMoved(MouseEvent e) {
275 //System.out.println("Moved");
276
277 // TODO Auto-generated method stub
278
279 }
280
281 public void mouseClicked (MouseEvent e) {
282 //System.out.println("Clicked");
283 }
284
285 public void mouseReleased (MouseEvent e) {
286 //System.out.println("Released");
287 draggingBox = false;
288 }
289
290 @Override
291 public void mouseEntered(MouseEvent e) {
292 //System.out.println("Entered");
293
294 // TODO Auto-generated method stub
295
296 }
297
298 @Override
299 public void mouseExited(MouseEvent e) {
300 //System.out.println("Exited");
301
302 // TODO Auto-generated method stub
303
304 }
305
306 @Override
307 public void mousePressed(MouseEvent e) {
308 //System.out.println("Pressed");
309 drawBox = true;
310 boxLeftTop = e.getPoint();
311 draggingBox = true;
312 // TODO Auto-generated method stub
313
314 }
[103]315}
Note: See TracBrowser for help on using the repository browser.