// main.cpp : Defines the entry point for the console application. // #include "config.h" #include #include #include #include #include #include "cximage/ximage.h" #include "xbrowseforfolder.h" // function prototypes bool CalculateDescriptors(const char *basedir); bool CategorizeDescriptors(const char *basedir); bool DetermineWinterSportSelect(); bool DetermineWinterSportBatch(const char *basedir); bool LoadAverages(const char *basedir); bool ShowAverages(const char *basedir); //#define TRAINING_DIR_DEBUG "C:\\Documents and Settings\\rvdzwet\\Desktop\\liacs\\MIR2010\\trainingSet" //#define TESTSET_DIR_DEBUG "C:\\Documents and Settings\\rvdzwet\\Desktop\\liacs\\MIR2010\\testSet" #define DEBUG 0 // Names of categories (folders) #define CATEGORY_SIZE 8 #define CATEGORY_1 "cat1.crowd" #define CATEGORY_2 "cat2.skijump" #define CATEGORY_3 "cat3.snowboarding" #define CATEGORY_4 "opt1.bobsleigh" #define CATEGORY_5 "opt2.icehockey" #define CATEGORY_6 "opt3.speedskating" #define CATEGORY_7 "opt4.downhillski" #define CATEGORY_8 "opt5.curling" // Nicely concat into a string array const char *categories[CATEGORY_SIZE] = { CATEGORY_1, CATEGORY_2, CATEGORY_3, CATEGORY_4, CATEGORY_5, CATEGORY_6, CATEGORY_7, CATEGORY_8}; // number of bins to use for each color // Note: number of bins must be a whole fraction of 256. if we would // use 256 bins for each color then a single histogram would be // 64MB, so we should choose a more sensible number #define BIN_COUNT 32 float average[CATEGORY_SIZE][BIN_COUNT*BIN_COUNT*BIN_COUNT]; // Find common coloured block shapes as characteristics // Walk over image with opt_spread steps, if pixel -within opt_tolerance- matches previous pixel // make the size of the specific block bigger. Blocks can be horizontally and vertically attached // Block '0' is used an non-matcher. Not taken into account for the time beeing. int opt_tolerance = 10; int opt_spread = 20; #define MAX_BLOCKSIZE 500 float average_block[CATEGORY_SIZE][MAX_BLOCKSIZE]; // ===========Block Interesting Results =========================== // opt_spread | opt_tolerance | total | (almost correct) // ------------|---------------|-------|--------------------------- // 10 | 10 | 20/41 | crowd, downhill, icehockey // 10 | 20 | 15/41 | crowd, downhill // 10 | 50 | 13/41 | crowd, downhill // 20 | 10 | 17/41 | crowd, icehockey, downhill // 20 | 20 | 17/41 | crowd, downhill, icehockey // 50 | 10 | 13/41 | // 50 | 20 | 14/41 | crowd, downhill // 50 | 50 | 12/41 | crowd, downhill // 100 | 100 | 10/41 | icehockey // Some prototyping foo on functions #define p_err(err_msg) printf("ERROR: %s\n", err_msg); #define _return(err_msg, retval) printf("DEBUG: %s\n",err_msg); cin.get(); return(retval); #define cout_status(msg,flag) cout << msg; (flag) ? cout << " off" : cout << " on"; cout << endl; #define cout_point(x,y) "[" << x << "," << y << "]" // Default options bool opt_debug = false; bool opt_verbose = false; bool opt_histogram = false; bool opt_block = true; bool file_exists(const char * filename) { if (FILE * file = fopen(filename, "r")) { fclose(file); return true; } return false; } int main(int argc, char **argv) { char dbdir[MAX_PATH]; char testdir[MAX_PATH]; #if defined(TRAINING_DIR_DEBUG) if (strcpy_s(dbdir, TRAINING_DIR_DEBUG) != 0) { _return("Cannot copy TRAINING_DIR_DEBUG",1); } if (strcpy_s(testdir, TESTSET_DIR_DEBUG) != 0) { _return("Cannot copy TESTSET_DIR_DEBUG",2); } #else // ask the user for the image database // in the image directory, the images are stored in categorized // folders. store the descriptors we calculate in the same // folder as the image if (XBrowseForFolder(NULL, "Please select image database folder", NULL, dbdir, sizeof(dbdir)) == FALSE) return 0; if (strlen(dbdir) == 0) return 0; if (XBrowseForFolder(NULL, "Please select image testset batch folder, could be any folder", NULL, testdir, sizeof(testdir)) == FALSE) return 0; if (strlen(testdir) == 0) return 0; #endif #if DEBUG if (!LoadAverages(dbdir)){ _return("Unable to load averages",1); } if (!DetermineWinterSportBatch(testdir)) { _return("could not run winter sport batch",1); } return(0); #endif // ask the user which option to use while (true) { //system("cls"); cout << "Using database directory: " << dbdir << endl; cout << "Using test directory: " << testdir << endl; cout << "Using categories: " << endl; for (int i = 0; i < CATEGORY_SIZE; i++) { cout << " - " << categories[i] << endl; } cout << "***************************" << endl; cout << "* Winter Olympic Imagery *" << endl; cout << "***************************" << endl; cout << endl; cout << "1. calculate descriptors" << endl; cout << "2. categorize descriptors (aka averages)" << endl; cout << "3. determine winter sport on single image" << endl; cout << "4. batch test winter sport" << endl; cout << "9. Show averages" << endl; cout << endl; cout_status("d. Turn debug mode", opt_debug); cout_status("v. Turn verbose mode", opt_verbose); cout_status("h. Turn histogram classifier", opt_histogram); cout_status("b. Turn block classifier", opt_block); cout << endl; cout << "==== Parameters Block Classifier ====" << endl; cout << "s. Set spread [currently: " << opt_spread << "]" << endl; cout << "t. Set Tolerance [currently: " << opt_tolerance << "]" << endl; cout << "Please select option, or type 'q' to quit: "; char c = _getch(); cout << c << endl; fflush(stdin); // start the chosen option switch (c) { case 'q': return 0; case '1': if (!CalculateDescriptors(dbdir)) { _return("could not calculate descriptors",1); } break; case '2': if (!CategorizeDescriptors(dbdir)){ _return("could not categorize descriptors",1); } break; case '3': if (!LoadAverages(dbdir)){ _return("Unable to load averages",1); } if (!DetermineWinterSportSelect()){ _return("could not determine winter sport",1); } break; case '4': if (!LoadAverages(dbdir)){ _return("Unable to load averages",1); } if (!DetermineWinterSportBatch(testdir)) { _return("could not run winter sport batch",1); } break; case '9': if (!LoadAverages(dbdir)){ _return("Unable to load averages",1); } if (!ShowAverages(testdir)) { _return("Unable to show averages",1); } break; case 'd': opt_debug = (!opt_debug); break; case 'v': opt_verbose = (!opt_verbose); break; case 'b': opt_block = (!opt_block); break; case 'h': opt_histogram = (!opt_histogram); break; case 's': cout << "Spread value: "; cin >> opt_spread; cout << "WARNING: Make sure to re-classify before (Batch)Testing" << endl; cout << "Any key to continue..." << endl; cin.get(); break; case 't': cout << "Tolerance value: "; cin >> opt_tolerance; cout << "WARNING: Make sure to re-classify before (Batch)Testing" << endl; cout << "Any key to continue..." << endl; cin.get(); break; default: continue; } } return 0; } // histogram should be a preallocated array of size BIN_COUNT*BIN_COUNT*BIN_COUNT elements and will // be filled with the color histogram of the image where path points at bool CalculateDescriptor(const char *path, float *histogram) { // load the image CxImage image(path, CXIMAGE_FORMAT_JPG); if (!image.IsValid()) return false; // clear histogram memset(histogram, 0, BIN_COUNT*BIN_COUNT*BIN_COUNT * sizeof(float)); // walk through the pixels to fill the histogram int width = (int)image.GetWidth(); int height = (int)image.GetHeight(); int bin_r, bin_g, bin_b; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { // Note: CxImage library starts counting at lower-left corner of the image, // which is seen as the top of the image. however, usually images // start counting from the top-left corner of the image. thus if you // want to get pixel(2, 2) from the top-left you would have to ask // for pixel (2, height - 2 - 1) from CxImage. although in this // situation we don't care which pixel is where, we only care about // its color. RGBQUAD rgb = image.BlindGetPixelColor(x, y, false); // determine the bin this color falls in bin_r = rgb.rgbRed / (256 / BIN_COUNT); bin_g = rgb.rgbGreen / (256 / BIN_COUNT); bin_b = rgb.rgbBlue / (256 / BIN_COUNT); histogram[bin_r*BIN_COUNT*BIN_COUNT + bin_g*BIN_COUNT + bin_b]++; } } // normalize the histogram so that all together the values will add up // to one. since there are width * height pixels, we divide each value // by this amount for (int i = 0; i < BIN_COUNT*BIN_COUNT*BIN_COUNT; i++) histogram[i] /= width * height; return true; } struct point { int x; int y; }; // histogram should be a preallocated array of size BIN_COUNT*BIN_COUNT*BIN_COUNT elements and will // be filled with the color histogram of the image where path points at bool CalculateBlock(const char *path, float *block) { // load the image CxImage image(path, CXIMAGE_FORMAT_JPG); if (!image.IsValid()) return false; // clear histogram memset(block, 0, MAX_BLOCKSIZE * sizeof(float)); // walk through the pixels to fill the histogram const int width = (int)image.GetWidth(); const int height = (int)image.GetHeight(); int * status = new int [width * height]; //memset(status,0, width * height * sizeof(int)); queue queue; int rgb_current, rgb_right, rgb_up = 0; int block_size = 0; point p, p_right, p_up; RGBQUAD rgb; // Pixel field status // 0 = No need for processing // 1 = Need processing // 2 = Done processing for (int x = 0; x < width; x += opt_spread) for (int y = 0; y < height; y += opt_spread) status[x * height + y] = 1; for (int x = 0; x < width; x += opt_spread) { for (int y = 0; y < height; y += opt_spread) { // Only process if required to and not yet done if (status[x * height + y] != 1) { if (opt_debug) cout << "Already processed point " << cout_point(x,y) << endl; continue; } if(opt_debug) cout << "Testing new point " << cout_point(x,y) << endl; // Stack based expantion and evaluation block_size = 0; p.x = x; p.y = y; queue.push(p); while (!queue.empty()) { // Note: CxImage library starts counting at lower-left corner of the image // which is seen as the top of the image. p = queue.front(); queue.pop(); // Make sure not to process points twice if (status[p.x * height + p.y] != 1) continue; if (opt_debug) cout << "Processing point " << cout_point(p.x,p.y) << endl; rgb = image.BlindGetPixelColor(p.x, p.y, false); rgb_current = (rgb.rgbRed + rgb.rgbBlue + rgb.rgbGreen); p_right.x = p.x + opt_spread; p_right.y = p.y; // Check if right one fits the bill if (p_right.x < width) { rgb = image.BlindGetPixelColor(p_right.x, p_right.y, false); rgb_right = (rgb.rgbRed + rgb.rgbBlue + rgb.rgbGreen); if (abs(rgb_right - rgb_current) < opt_tolerance) { if (opt_debug) cout << "Right point " << cout_point(p_right.x,p_right.y) << " gets included in block" << endl; block_size++; queue.push(p_right); } } // Check if up one fits the bill p_up.x = p.x; p_up.y = p.y + opt_spread; if (p_up.y < height) { rgb = image.BlindGetPixelColor(p_up.x, p_up.y, false); rgb_up = (rgb.rgbRed + rgb.rgbBlue + rgb.rgbGreen); if (abs(rgb_up - rgb_current) < opt_tolerance) { if (opt_debug) cout << "Upper point " << cout_point(p_up.x, p_up.y) << " gets included in block" << endl; block_size++; queue.push(p_up); } } status[p.x * height + p.y] = 2; } if (opt_debug) cout << cout_point(x,y) << " blocksize " << block_size << endl; if (block_size > MAX_BLOCKSIZE) { cout << "WARN: block_size larger than fixed limit of " << MAX_BLOCKSIZE << endl; block_size = MAX_BLOCKSIZE; } block[block_size]++; } } delete status; return true; } bool CalculateDescriptors(const char *basedir) { // the histogram that we reuse for each image float *histogram = new float[BIN_COUNT*BIN_COUNT*BIN_COUNT]; float *block = new float[MAX_BLOCKSIZE]; // walk through all images // Note: each of the three categories has 50 images char path[MAX_PATH]; char catdir[MAX_PATH]; const char *catname; FILE *file = NULL; for (int c = 0; c < CATEGORY_SIZE; c++) { catname = categories[c]; sprintf(catdir, "%s\\%s\\", basedir, catname); cout << "[" << catname << "] Using directory " << catdir << endl; // process the images in the directory for (int i = 1; i <= 50; i++) { SAFE_SPRINTF(path, sizeof(path), "%s%i.jpg", catdir, i); if (!file_exists(path)) { continue; } cout << "[" << catname << "] processing image " << i << endl; // calculate the histogram descriptor if (!CalculateDescriptor(path, histogram)) goto failure; if (!CalculateBlock(path, block)) goto failure; if (opt_verbose) { for (int u = 0; u < MAX_BLOCKSIZE; u++) { if (block[u]) cout << "Blocksize " << u << " : " << block[u] << endl; } } // save the descriptor,block to disk SAFE_SPRINTF(path, sizeof(path), "%s%i.dat", catdir, i); if ((file = fopen(path, "wb")) == NULL) goto failure; if (fwrite(histogram, sizeof(float), BIN_COUNT*BIN_COUNT*BIN_COUNT, file) != BIN_COUNT*BIN_COUNT*BIN_COUNT) goto failure; if (fwrite(block, sizeof(float), MAX_BLOCKSIZE, file) != MAX_BLOCKSIZE) goto failure; SAFE_CLOSEFILE(file); } } // release resources SAFE_DELETE_ARRAY(histogram); SAFE_DELETE_ARRAY(block); return true; failure: SAFE_CLOSEFILE(file); SAFE_DELETE_ARRAY(histogram); SAFE_DELETE_ARRAY(block); return false; } bool CategorizeDescriptors(const char *basedir) { // analyze the descriptors per category to determine the // characteristics of that category float *histogram = new float[BIN_COUNT*BIN_COUNT*BIN_COUNT]; float *block = new float[MAX_BLOCKSIZE]; float *average_block = new float[MAX_BLOCKSIZE]; float *average = new float[BIN_COUNT*BIN_COUNT*BIN_COUNT]; // walk through all descriptors char path[MAX_PATH]; char catdir[MAX_PATH]; const char *catname; FILE *file = NULL; int c_size = 0; for (int c = 0; c < CATEGORY_SIZE; c++) { c_size = 0; catname = categories[c]; sprintf(catdir,"%s\\%s\\", basedir, catname); // Make sure initial values are 0 memset(average, 0, BIN_COUNT*BIN_COUNT*BIN_COUNT * sizeof(float)); memset(average_block, 0, MAX_BLOCKSIZE * sizeof(float)); // average all descriptors for (int i = 1; i <= 50; i++) { SAFE_SPRINTF(path, sizeof(path), "%s%i.dat", catdir, i); if (!file_exists(path)) { p_err("File does not exists"); continue; } cout << "[" << catname << "] processing image " << i << endl; // load the histogram descriptor if ((file = fopen(path, "rb")) == NULL) { p_err("Cannot open average datafile"); goto failure; } if (fread(histogram, sizeof(float), BIN_COUNT*BIN_COUNT*BIN_COUNT, file) != BIN_COUNT*BIN_COUNT*BIN_COUNT) { p_err("Cannot read histogram"); goto failure; } if (fread(block, sizeof(float), MAX_BLOCKSIZE, file) != MAX_BLOCKSIZE) { p_err("Cannot read block"); goto failure; } SAFE_CLOSEFILE(file); // add the value of each bin to the average for (int b = 0; b < BIN_COUNT*BIN_COUNT*BIN_COUNT; b++) average[b] += histogram[b]; for (int b = 0; b < MAX_BLOCKSIZE; b++) average_block[b] += block[b]; c_size++; } for (int b = 0; b < BIN_COUNT*BIN_COUNT*BIN_COUNT; b++) average[b] /= c_size; for (int b = 0; b < MAX_BLOCKSIZE; b++) { average_block[b] /= c_size; } // save the average to disk SAFE_SPRINTF(path, sizeof(path), "%s%s.dat", catdir, "average"); if ((file = fopen(path, "wb")) == NULL) goto failure; if (fwrite(average, sizeof(float), BIN_COUNT*BIN_COUNT*BIN_COUNT, file) != BIN_COUNT*BIN_COUNT*BIN_COUNT) goto failure; if (fwrite(average_block, sizeof(float), MAX_BLOCKSIZE, file) != MAX_BLOCKSIZE) goto failure; SAFE_CLOSEFILE(file); } // release resources SAFE_DELETE_ARRAY(histogram); SAFE_DELETE_ARRAY(average); SAFE_DELETE_ARRAY(block); return true; failure: SAFE_CLOSEFILE(file); SAFE_DELETE_ARRAY(histogram); SAFE_DELETE_ARRAY(block); return false; } bool LoadAverages(const char *basedir) { /* determine the distance to each category */ const char *catname; char catdir[MAX_PATH]; char path[MAX_PATH]; FILE *file = NULL; for (int c = 0; c < CATEGORY_SIZE; c++) { catname = categories[c]; sprintf(catdir, "%s\\%s\\", basedir, catname); // load the average from disk SAFE_SPRINTF(path, sizeof(path), "%s%s.dat", catdir, "average"); if ((file = fopen(path, "rb")) == NULL) { cout << "Cannot open " << path << endl; return false; } if (fread(average[c], sizeof(float), BIN_COUNT*BIN_COUNT*BIN_COUNT, file) != BIN_COUNT*BIN_COUNT*BIN_COUNT) return false; if (fread(average_block[c], sizeof(float), MAX_BLOCKSIZE, file) != MAX_BLOCKSIZE) return false; SAFE_CLOSEFILE(file); } return true; } bool ShowAverages(const char *basedir) { if (opt_histogram) { cout << "Histogram averages" << endl; for (int c = 0; c < CATEGORY_SIZE; c++) { cout << "Category [" << categories[c] << "] " << endl; for (int i = 1; i <= BIN_COUNT*BIN_COUNT*BIN_COUNT; i++) { cout << fixed << setprecision(2) << average[c][i-1] << " "; if ((i % 10) == 0) cout << endl; } cout << endl; } } if (opt_block) { cout << "Block averages" << endl; for (int c = 0; c < CATEGORY_SIZE; c++) { cout << "Category [" << categories[c] << "] " << endl; for (int i = 1; i < MAX_BLOCKSIZE; i++) { cout << fixed << setprecision(2) << average_block[c][i-1] << " "; if ((i % 10) == 0) cout << endl; } cout << endl; } } return true; } int DetermineCategory(const char *path, const int guess=-1, const bool verbose=false) { float *histogram = new float[BIN_COUNT*BIN_COUNT*BIN_COUNT]; float *block = new float[MAX_BLOCKSIZE]; float cat2dist[CATEGORY_SIZE]; float cat2block[CATEGORY_SIZE]; /* First category default best canidate */ int cat_histogram = 0; int cat_block = 0; /* calculate the histogram of the image */ if (!CalculateDescriptor(path, histogram)) return -1; if (!CalculateBlock(path, block)) return -1; /* determine the distance to each category */ for (int c = 0; c < CATEGORY_SIZE; c++) { // determine distance cat2dist[c] = 0.0f; cat2block[c] = 0.0f; for (int b = 0; b < BIN_COUNT*BIN_COUNT*BIN_COUNT; b++) cat2dist[c] += fabs(histogram[b] - average[c][b]); /* No including of the non-matching points currently */ for (int b = 1; b < MAX_BLOCKSIZE; b++) cat2block[c] += fabs(block[b] - average_block[c][b]); } /* determine the winning category */ for (int i = 1; i < CATEGORY_SIZE; i++) { if (cat2dist[i] < cat2dist[cat_histogram]) cat_histogram = i; if (cat2block[i] < cat2block[cat_block]) cat_block = i; } if (verbose) { /* Dirty hack to show some more details in case of failure */ if (opt_histogram && guess != -1 && guess != cat_histogram) { for (int i = 0; i < CATEGORY_SIZE; i++) { printf("%s [histogram] distance to %-20s: %f %s\n", (cat_histogram == i) ? "*" : " ", categories[i], cat2dist[i],(cat_histogram == i) ? "*" : ""); } } if (opt_block && guess != -1 && guess != cat_block) { for (int i = 0; i < CATEGORY_SIZE; i++) { printf("%s [block] distance to %-20s: %f %s\n", (cat_block == i) ? "*" : " ", categories[i], cat2block[i],(cat_block == i) ? "*" : ""); } } } /* return result */ if (opt_histogram) { return cat_histogram; } else if (opt_block) { return cat_block; } else { return -1; } } /* ask for an input image and determine the most likely * category it belongs to */ bool DetermineWinterSportSelect() { float *histogram = new float[BIN_COUNT*BIN_COUNT*BIN_COUNT]; char path[MAX_PATH] = {0}; char catdir[MAX_PATH] = {0}; const char *catname = NULL; FILE *file = NULL; int c = NULL; float cat2dist[CATEGORY_SIZE] = {0}; OPENFILENAME ofn = {0}; //ofn.lpstrFilter = "Image files\0*.jpg;*.png;*.bmp\0\0"; ofn.lpstrFilter = "Image files\0*.jpg\0\0"; ofn.lpstrFile = path; ofn.nMaxFile = MAX_PATH; ofn.lpstrTitle = "Choose image file"; ofn.Flags = OFN_PATHMUSTEXIST | OFN_FILEMUSTEXIST; ofn.lStructSize = sizeof(OPENFILENAME); if (GetOpenFileName(&ofn) == FALSE) goto failure; if ((c = DetermineCategory(path,-2)) == -1) return false; cout << "The category this image belongs is category: " << categories[c] << " [" << c << "]" << endl; cout << "Press any key to continue... "; _getch(); fflush(stdin); // release resources SAFE_DELETE_ARRAY(histogram); return true; failure: SAFE_CLOSEFILE(file); SAFE_DELETE_ARRAY(histogram); return false; } bool DetermineWinterSportBatch(const char *basedir) { const char *catname; char catdir[MAX_PATH]; char path[MAX_PATH]; int all_total = 0; int all_succes = 0; int c_total[CATEGORY_SIZE] = {0}; int c_succes[CATEGORY_SIZE] = {0}; /* determine the distance to each category */ for (int c = 0; c < CATEGORY_SIZE; c++) { catname = categories[c]; sprintf(catdir, "%s\\%s\\", basedir, catname); /* process the images in the directory */ for (int i = 1; i <= 50; i++) { SAFE_SPRINTF(path, sizeof(path), "%s%i.jpg", catdir, i); if (!file_exists(path)) { continue; } c_total[c]++; /* Check if file matches category */ if (DetermineCategory(path,c) == c) { cout << "[" << catname << "] testing image " << i << " : OK" << endl; c_succes[c]++; } else { cout << "[" << catname << "] testing image " << i << " : FAIL" << endl; DetermineCategory(path,c,opt_verbose); } } cout << "[" << catname << "] results " << c_succes[c] << "/" << c_total[c] << endl; } /* Display grand total */ cout << "=== Totals ===" << endl; cout << "Clasifier used: "; if (opt_histogram) cout << "histogram"; else if (opt_block) cout << "block"; cout << endl; for (int c = 0; c < CATEGORY_SIZE; c++) { catname = categories[c]; printf ("[%-20s] %i/%i\n",catname,c_succes[c],c_total[c]); all_total += c_total[c]; all_succes += c_succes[c]; } printf ("[%-20s] %i/%i\n","total",all_succes,all_total); cout << "Press any key to continue..."; cin.get(); return true; }