Change Threads for Process to new processors. Change messages for explain...

Change Threads for Process to new processors. Change messages for explain problems. Show process in console. Fix problem NaN in ARFF

.gitignore

 *.pyc
 data/*
+models_checkpoints/*
+venv/*
 !data/demo.jpg
 !data/pynovisao.png
 !data/demo/.gitignore

src/extraction/feature_extraction.py

@@ -11,8 +11,8 @@
 import io
 import itertools
 import os
-import threading
-
+import multiprocessing 
+from multiprocessing import Process, Manager
 from interface.interface import InterfaceException as IException
 
 from util.file_utils import File
@@ -23,7 +23,9 @@ from extractor import Extractor
 from tqdm import tqdm
 import sys
 
-
+if not sys.warnoptions:
+    import warnings
+    warnings.simplefilter("ignore")
 class FeatureExtractor(object):
     """Handle the feature extraction."""
 
@@ -40,13 +42,11 @@ class FeatureExtractor(object):
         self.tkParent=tkParent
 
     def extract_all(self, dataset, output_file=None, dirs=None, overwrite=True):
-        self.labels = []
-        self.types = []
-
-        self.data = []
+  
+        self.data = Manager().list() #is a necessary because have a problem with use Process and normaly declaration
         self.threads = []
-        self.labels = []
-        self.types = []
+        self.labels = Manager().list()
+        self.types = Manager().list()
         """Runs the feature extraction algorithms on all images of dataset.
 
         Parameters
@@ -101,6 +101,7 @@ class FeatureExtractor(object):
         with tqdm(total=len(self.threads)) as pbar:
             for  t in self.threads:
                  t.start()
+                 pbar.update(1)
             pbar.close()
 
         self.print_console("Waiting for workers to finish extracting attributes from images!")
@@ -108,7 +109,6 @@ class FeatureExtractor(object):
             for t in self.threads:
                 t.join()
                 ppbar.update(1)
-
             ppbar.close()
         self.print_console("The process was completed with "+str(len(self.threads))+" images!")
         if len(self.data) == 0:
@@ -116,7 +116,7 @@ class FeatureExtractor(object):
 
         # Save the output file in ARFF format
         # self._save_output(File.get_filename(dataset), classes, self.labels, self.types, self.data, output_file)
-        self._save_output(File.get_filename(dataset), classes, self.labels, self.types, self.data, output_file)
+        self._save_output(File.get_filename(dataset), classes, self.labels[0], self.types[0], self.data, output_file)
         end_time = TimeUtils.get_time()
 
         return output_file, (end_time - start_time)
@@ -130,9 +130,8 @@ class FeatureExtractor(object):
         for  item in  items :
             if item.startswith('.'):
                 continue
-
-            th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
-
+            #th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
+            th = multiprocessing.Process(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
             self.threads.append(th)
 
 
@@ -149,14 +148,17 @@ class FeatureExtractor(object):
         if len(self.data) > 0:
             values = list(
                 itertools.chain.from_iterable(zip(*([extractor().run(image) for extractor in self.extractors]))[2]))
+	    
             self.data.append(values + [cl if cl in classes else classes[0]])
+	    
         else:
-            self.labels, self.types, values = [list(itertools.chain.from_iterable(ret))
+            labs, tys, values = [list(itertools.chain.from_iterable(ret))
                                                for ret in
                                                zip(*(extractor().run(image) for extractor in self.extractors))]
+            self.labels.append(labs)
+            self.types.append(tys)
             self.data.append(values + [cl if cl in classes else classes[0]])
 
-
     def extract_one_file(self, dataset, image_path, output_file=None):
         """Runs the feature extraction algorithms on specific image.
 

src/extraction/hog.py

@@ -8,6 +8,11 @@
     
     Name: hog.py
     Author: Alessandro dos Santos Ferreira ( santosferreira.alessandro@gmail.com )
+	
+    Change parameter Visualise for Visualize because is deprecaded
+    Date:02/01/2019
+    Author: Diego Andre Sant Ana		 	
+		
 """
 from skimage import feature
 
@@ -36,13 +41,13 @@ class HOG(Extractor):
         features : tuple
             Returns a tuple containing a list of labels, type and values for each feature extracted.
         """
-        image_grayscale = ImageUtils.image_grayscale(image, bgr = True)
+        image_grayscale = ImageUtils.image_grayscale(image, bgr=True)
         image_128x128 = ImageUtils.image_resize(image_grayscale, 128, 128)
-        
+
         values, _ = feature.hog(image_128x128, orientations=8, pixels_per_cell=(32, 32),
-                    cells_per_block=(1, 1), visualise=True)
-                    
-        labels = [m+n for m,n in zip(['hog_'] * len(values),map(str,range(0,len(values))))]
+                                cells_per_block=(1, 1), visualise=True)
+
+        labels = [m + n for m, n in zip(['hog_'] * len(values), map(str, range(0, len(values))))]
         types = [Extractor.NUMERIC] * len(labels)
 
         return labels, types, list(values)

src/extraction/image_moments.py

@@ -16,7 +16,7 @@ import cv2
 from util.utils import ImageUtils
 from skimage.measure import regionprops, moments, moments_central
 from skimage.morphology import label
-
+import numpy as np
 from extractor import Extractor
 
 class RawCentralMoments(Extractor):
@@ -53,8 +53,8 @@ class RawCentralMoments(Extractor):
         
         row = m[0, 1] / m[0, 0]
         col = m[1, 0] / m[0, 0]
-
-        mu = measure.moments_central(image_binary, row, col)
+	
+        mu = measure.moments_central(image_binary, center=(row, col), order=3)
         
         values_mu = [mu[p, q] for (p, q) in self._moments_order]
         labels_mu = [M+str(p)+str(q) for M,(p,q) in zip(['Mu_'] * len(self._moments_order), self._moments_order)]
@@ -104,8 +104,9 @@ class HuMoments(Extractor):
         image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
 
         values_hu= cv2.HuMoments(cv2.moments(image)).flatten()
+        values_hu = list(values_hu)
 
-
+        values_hu= np.nan_to_num(values_hu)
         labels_hu = [m+n for m,n in zip(['Hu_'] * len(values_hu),map(str,range(0,len(values_hu))))]
         
         labels = labels_hu

src/util/split_convertall.sh

+#!/bin/bash
+
+#
+#    Script - convert tif to png
+#
+#    Name: script_convertall.sh
+#    Author: Gabriel Kirsten Menezes (gabriel.kirsten@hotmail.com)
+#
+
+echo "[SCRIPT CONVERT ALL] Initializing..."
+
+dir_train="../../data/demo_split/train"
+dir_validation="../../data/demo_split/validation"
+
+echo "[SCRIPT CONVERT ALL] Converting train..."
+
+for dir_class in `ls $dir_train`;
+do
+    echo "[SCRIPT CONVERT ALL] Converting class -" $dir_class;
+    convert $dir_train/$dir_class/* $dir_train/$dir_class/$dir_class.png
+    echo "[SCRIPT CONVERT ALL] Removing all .tif files in $dir_class ..."
+    rm $dir_train/$dir_class/*.tif
+done
+
+echo "[SCRIPT CONVERT ALL] Converting validation..."
+
+for dir_class in `ls $dir_validation`;
+do
+    echo "[SCRIPT CONVERT ALL] Converting class -" $dir_class;
+    convert $dir_validation/$dir_class/* $dir_validation/$dir_class/$dir_class.png
+    echo "[SCRIPT CONVERT ALL] Removing all .tif files in $dir_class ..."
+    rm $dir_validation/$dir_class/*.tif
+done
+
+echo "[SCRIPT CONVERT ALL] OK! DONE."