source: java-automator/src/nl/rickvanderzwet/farmville/ScreenRecognition.java@ 111

Last change on this file since 111 was 111, checked in by Rick van der Zwet, 15 years ago

Hacking with overlays and detections
File size: 8.4 KB
Line 
1/*
2 * Rick van der Zwet <info@rickvanderzwet.nl>
3 * License BSDLike http://rickvanderzwet.nl/LICENSE
4 * XXX: Do not look at this code, it's dirty and _NOT_ the way Java should be written.
5 */
6
7package nl.rickvanderzwet.farmville;
8
9import java.awt.*;
10import java.awt.event.*;
11import java.awt.image.BufferedImage;
12import java.awt.image.RasterFormatException;
13
14import javax.swing.BorderFactory;
15import javax.swing.JButton;
16import javax.swing.JFrame;
17import javax.swing.JLabel;
18import javax.swing.JLayeredPane;
19import javax.swing.JPanel;
20import javax.swing.Timer;
21
22//class ScreenPainter extends Panel {
23// private static final long serialVersionUID = -3073797542841857452L;
24// BufferedImage foo = null;
25//
26// public void paint(Graphics g) {
27// // System.out.println("FOO");
28// // g.drawImage(foo,20, 50, this);
29// }
30//}
31
32public class ScreenRecognition extends JFrame implements ActionListener,MouseMotionListener,MouseListener {
33 /**
34 *
35 */
36 private static final long serialVersionUID = -6742715532913330783L;
37 static int mouseSurround = 120;
38 Image image;
39 static Color pixel;
40 static Point mouse;
41 static BufferedImage bi;
42 static Robot bot;
43
44 static boolean freezeImage = false;
45 static boolean mouseClickEnabled = false;
46
47 static boolean foundPlowLocation = false;
48
49 static Button redButton;
50 static Button greenButton;
51
52 Point boxLeftTop, boxRightBottom = new Point (0,0);
53 Boolean drawBox = false;
54 static Boolean draggingBox = false;
55
56 private static void pressMouse() throws InterruptedException {
57 if (mouseClickEnabled) {
58 bot.mousePress(InputEvent.BUTTON1_MASK);
59 bot.mouseRelease(InputEvent.BUTTON1_MASK);
60 }
61
62 }
63
64 public ScreenRecognition() {
65 setTitle("Screen Recognition Example");
66 JLayeredPane layeredPane = new JLayeredPane();
67
68 //layeredPane.setBorder(BorderFactory.createTitledBorder("Foo"));
69 //Panel foo = new ScreenPainter();
70 //foo.setBackground(Color.RED);
71 //foo.setBounds(70, 70, 50, 50);
72
73 JLabel bar = new JLabel();
74 bar.setText("FooBAR");
75 bar.setBounds(0, 0, 200, 20);
76 // Create 3 buttons
77 JPanel top = new JPanel();
78 top.setBackground(Color.white);
79 top.setBounds(20, 20, 50, 50);
80 JPanel middle = new JPanel();
81 middle.setBackground(Color.gray);
82 middle.setBounds(40, 40, 50, 50);
83 JPanel bottom = new JPanel();
84 bottom.setBackground(Color.black);
85 bottom.setBounds(60, 60, 50, 50);
86
87 // Place the buttons in different layers
88// layeredPane.add(foo, new Integer(5));
89// layeredPane.add(bar, new Integer(5));
90// layeredPane.add(middle,new Integer(0));
91// layeredPane.add(top, new Integer(4));
92// layeredPane.add(bottom, new Integer(0));
93
94 Panel buttons = new Panel();
95 redButton = new Button("Red");
96 Button blueButton = new Button("Blue");
97 greenButton = new Button("Green");
98 redButton.setName("redButton");
99 redButton.setLabel("Freeze Screen");
100 redButton.addActionListener(this);
101 blueButton.setName("blueButton");
102 blueButton.setLabel("Stop clicking");
103 blueButton.addActionListener(this);
104 buttons.add(redButton);
105 buttons.add(blueButton);
106 buttons.add(greenButton);
107
108
109// add(layeredPane);
110 add(buttons,"South");
111
112
113 setSize(400,300);
114 addWindowListener(new WindowAdapter() {
115 public void windowClosing(WindowEvent e){
116 System.exit(0);
117 }
118 });
119 setVisible(true);
120 addMouseMotionListener( this );
121 addMouseListener( this );
122
123 }
124
125 public void actionPerformed(ActionEvent event) {
126 System.out.println("ActionPerformed");
127 Component c = (Component)event.getSource();
128 if (c.getName() == "redButton") {
129 if (freezeImage) {
130 freezeImage = false;
131 redButton.setLabel("Freeze image");
132 } else {
133 c.getToolkit().beep();
134
135 /* Freeze screen after number of seconds seconds */
136 ActionListener freezeScreen = new ActionListener() {
137 public void actionPerformed(ActionEvent evt) {
138 System.out.println(freezeImage);
139 freezeImage = true;
140 redButton.setLabel("Release image");
141 }
142 };
143 Timer foo = new Timer(3000, freezeScreen);
144 foo.setRepeats(false);
145 foo.start();
146 redButton.setLabel("Freezing image...");
147 }
148 } else if (c.getName() == "blueButton") {
149 mouseClickEnabled = !mouseClickEnabled;
150 if (mouseClickEnabled) {
151 ((Button) c).setLabel("Stop clicking");
152 } else {
153 ((Button) c).setLabel("Start clicking");
154 }
155 }
156 }
157
158 public void paint(Graphics g) {
159 super.paint(g);
160 g.drawString("Mouse:" + mouse.x + "," + mouse.y ,20,40);
161 g.drawString("PixelRGB:" + pixel.toString() ,150,40);
162
163 g.drawRect(18, 48, mouseSurround * 2 + 4, mouseSurround * 2 + 4);
164 g.setColor(pixel);
165 g.fillRect(300,100,20,20);
166
167 g.drawImage(bi,20, 50, this);
168
169 if (drawBox){
170 g.setColor(Color.RED);
171 g.drawRect(boxLeftTop.x, boxLeftTop.y, Math.abs(boxRightBottom.x - boxLeftTop.x), Math.abs(boxRightBottom.y - boxLeftTop.y));
172 }
173
174 greenButton.setEnabled(foundPlowLocation);
175
176 bi.createGraphics();
177
178 }
179
180 public static void main(String[] args) throws AWTException, InterruptedException {
181 //bot.mouseMove(100, 100);
182 bot = new Robot();
183 ScreenRecognition foo = new ScreenRecognition();
184
185 Point prevLocation = new Point(0,0);
186 while (true) {
187 PointerInfo pointer = MouseInfo.getPointerInfo();
188 mouse = pointer.getLocation();
189 pixel = bot.getPixelColor(mouse.x, mouse.y);
190 try {
191 if (!freezeImage && !mouse.equals(prevLocation)) {
192 System.out.println("Location mouse changed " + mouse + "(was: " + prevLocation + ")");
193 prevLocation = mouse;
194 bi=bot.createScreenCapture(new Rectangle(mouse.x + mouseSurround,
195 mouse.y + mouseSurround));
196 bi = bi.getSubimage(mouse.x - mouseSurround,
197 mouse.y - mouseSurround,
198 mouseSurround * 2, mouseSurround * 2);
199
200 /* See is enabled */
201 final Color target = new Color(255, 170, 0);
202 int pixelTreshold = 10;
203
204 search: for (int x = 0; x < bi.getWidth(); x++) {
205 for (int y = 0; y < bi.getHeight(); y++){
206 if (bi.getRGB(x, y) == target.getRGB()) {
207 pixelTreshold--;
208 if (pixelTreshold == 0) {
209 break search;
210 }
211 }
212 }
213 }
214
215 /* Select text only */
216 final Color white = new Color(255, 255, 255);
217 final Color black = new Color(0, 0, 0);
218 for (int x = 0; x < bi.getWidth(); x++) {
219 for (int y = 0; y < bi.getHeight(); y++){
220 if (bi.getRGB(x, y) != white.getRGB()) {
221 bi.setRGB(x,y, black.getRGB());
222 }
223 }
224 }
225
226 /* Determine text borders */
227 Point leftTop = new Point(bi.getWidth(),bi.getHeight());
228 Point rightBottom = new Point(0,0);
229 for (int x = 0; x < bi.getWidth(); x++) {
230 for (int y = 0; y < bi.getHeight(); y++){
231 if (bi.getRGB(x, y) == white.getRGB()) {
232 if (x < leftTop.x && y < leftTop.y) {
233 leftTop.x = x;
234 leftTop.y = y;
235 }
236 if (x > rightBottom.x && y > rightBottom.y) {
237 rightBottom.x = x;
238 rightBottom.y = y;
239 }
240 }
241 }
242 }
243
244 /* Do basic pattern recognition */
245
246
247 if (pixelTreshold == 0) {
248 pressMouse();
249 System.out.println("Button Click at [" + mouse.x + "," + mouse.y + "]");
250 }
251
252 }
253 foo.repaint();
254 } catch (IllegalArgumentException e) {
255 continue;
256 } catch (RasterFormatException e) {
257 continue;
258 }
259 // Allow time to focus Firefox window
260 Thread.sleep(100);
261 }
262
263 }
264
265
266 @Override
267 public void mouseDragged(MouseEvent e) {
268 boxRightBottom = e.getPoint();
269 //System.out.println("Dragged");
270
271 }
272
273 @Override
274 public void mouseMoved(MouseEvent e) {
275 //System.out.println("Moved");
276
277 // TODO Auto-generated method stub
278
279 }
280
281 public void mouseClicked (MouseEvent e) {
282 //System.out.println("Clicked");
283 }
284
285 public void mouseReleased (MouseEvent e) {
286 //System.out.println("Released");
287 draggingBox = false;
288 }
289
290 @Override
291 public void mouseEntered(MouseEvent e) {
292 //System.out.println("Entered");
293
294 // TODO Auto-generated method stub
295
296 }
297
298 @Override
299 public void mouseExited(MouseEvent e) {
300 //System.out.println("Exited");
301
302 // TODO Auto-generated method stub
303
304 }
305
306 @Override
307 public void mousePressed(MouseEvent e) {
308 //System.out.println("Pressed");
309 drawBox = true;
310 boxLeftTop = e.getPoint();
311 draggingBox = true;
312 // TODO Auto-generated method stub
313
314 }
315}
Note: See TracBrowser for help on using the repository browser.