Merge branch 'master' of http://git.inovisao.ucdb.br/inovisao/pynovisao into feature/pseudo-label

.gitignore

 *.pyc
 data/*
+models_checkpoints/*
+venv/*
 !data/demo.jpg
 !data/pynovisao.png
 !data/demo/.gitignore

INSTALL.sh

@@ -17,16 +17,16 @@ fi
 echo "[PYNOVISAO INSTALLER] Updating apt-get..."
 sudo apt-get -qq update
 echo "[PYNOVISAO INSTALLER] Installing prerequisites..."
-sudo apt-get -qq -y install libfreetype6-dev tk tk-dev python-pip default-jre default-jdk weka weka-doc python-tk python-matplotlib python-opencv
-
+sudo apt-get -qq -y install libfreetype6-dev tk tk-dev python-pip openjdk-8-jre openjdk-8-jdk weka weka-doc python-tk python-matplotlib
+source ~/.bashrc
 
 echo "[PYNOVISAO INSTALLER] Upgrading pip..."
-pip install --upgrade pip  --quiet
+sudo pip install --upgrade pip  --quiet
 # Numpy must be installed before installing javabridge
 echo "[PYNOVISAO INSTALLER] Installing numpy..."
-pip install numpy --quiet
+sudo pip install numpy==1.14.5 --quiet
 echo "[PYNOVISAO INSTALLER] Installing libraries..."
-pip install -r requeriments.txt --quiet
+sudo pip install -r requeriments.txt --quiet
 
 printf "${YELLOW} \n======== WARNING ========\n"
 printf "The Keras is necessary so that it is possible to use CNN. It is recommended to install the version for GPU processing (if available) but it is also possible to use CPU processing.\n"

README.md

 # Título: Pynovisao
-## Autores (ordem alfabética): Adair da Silva Oliveira Junior, Alessandro dos Santos Ferreira, Diego André Sant'Ana(diegoandresantana@gmail.com),Everton Castelão Tetila(evertontetila@gmail.com), Gabriel Kirsten Menezes(gabriel.kirsten@hotmail.com), Hemerson Pistori (pistori@ucdb.br), Nícolas Alessandro de Souza Belete(nicolas.belete@gmail.com)
+## Autores (ordem alfabética): Adair da Silva Oliveira Junior, Alessandro dos Santos Ferreira, Diego André Sant'Ana(diegoandresantana@gmail.com), Diogo Nunes Gonçalves(dnunesgoncalves@gmail.com), Everton Castelão Tetila(evertontetila@gmail.com), Felipe Silveira(eng.fe.silveira@gmail.com), Gabriel Kirsten Menezes(gabriel.kirsten@hotmail.com), Gilberto Astolfi(gilbertoastolfi@gmail.com), Hemerson Pistori (pistori@ucdb.br), Nícolas Alessandro de Souza Belete(nicolas.belete@gmail.com)
 
 ## Resumo:
 

requeriments.txt

@@ -7,15 +7,16 @@ sympy
 nose
 networkx
 scikit-image
-javabridge  
+javabridge==1.0.18  
 python-weka-wrapper
 cycler==0.10
 cython
 h5py
 scikit-learn
-tensorflow-gpu
 statistics
 pandas_ml
 pyxdg
 opencv-contrib-python
 python-interface
+tqdm
+sklearn

src/extraction/feature_extraction.py

@@ -11,8 +11,8 @@
 import io
 import itertools
 import os
-import threading
-
+import multiprocessing 
+from multiprocessing import Process, Manager
 from interface.interface import InterfaceException as IException
 
 from util.file_utils import File
@@ -23,7 +23,9 @@ from extractor import Extractor
 from tqdm import tqdm
 import sys
 
-
+if not sys.warnoptions:
+    import warnings
+    warnings.simplefilter("ignore")
 class FeatureExtractor(object):
     """Handle the feature extraction."""
 
@@ -40,13 +42,11 @@ class FeatureExtractor(object):
         self.tkParent=tkParent
 
     def extract_all(self, dataset, output_file=None, dirs=None, overwrite=True):
-        self.labels = []
-        self.types = []
-
-        self.data = []
+  
+        self.data = Manager().list() #is a necessary because have a problem with use Process and normaly declaration
         self.threads = []
-        self.labels = []
-        self.types = []
+        self.labels = Manager().list()
+        self.types = Manager().list()
         """Runs the feature extraction algorithms on all images of dataset.
 
         Parameters
@@ -101,6 +101,7 @@ class FeatureExtractor(object):
         with tqdm(total=len(self.threads)) as pbar:
             for  t in self.threads:
                  t.start()
+                 pbar.update(1)
             pbar.close()
 
         self.print_console("Waiting for workers to finish extracting attributes from images!")
@@ -108,7 +109,6 @@ class FeatureExtractor(object):
             for t in self.threads:
                 t.join()
                 ppbar.update(1)
-
             ppbar.close()
         self.print_console("The process was completed with "+str(len(self.threads))+" images!")
         if len(self.data) == 0:
@@ -116,7 +116,7 @@ class FeatureExtractor(object):
 
         # Save the output file in ARFF format
         # self._save_output(File.get_filename(dataset), classes, self.labels, self.types, self.data, output_file)
-        self._save_output(File.get_filename(dataset), classes, self.labels, self.types, self.data, output_file)
+        self._save_output(File.get_filename(dataset), classes, self.labels[0], self.types[0], self.data, output_file)
         end_time = TimeUtils.get_time()
 
         return output_file, (end_time - start_time)
@@ -130,9 +130,8 @@ class FeatureExtractor(object):
         for  item in  items :
             if item.startswith('.'):
                 continue
-
-            th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
-
+            #th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
+            th = multiprocessing.Process(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
             self.threads.append(th)
 
 
@@ -149,14 +148,17 @@ class FeatureExtractor(object):
         if len(self.data) > 0:
             values = list(
                 itertools.chain.from_iterable(zip(*([extractor().run(image) for extractor in self.extractors]))[2]))
+	    
             self.data.append(values + [cl if cl in classes else classes[0]])
+	    
         else:
-            self.labels, self.types, values = [list(itertools.chain.from_iterable(ret))
+            labs, tys, values = [list(itertools.chain.from_iterable(ret))
                                                for ret in
                                                zip(*(extractor().run(image) for extractor in self.extractors))]
+            self.labels.append(labs)
+            self.types.append(tys)
             self.data.append(values + [cl if cl in classes else classes[0]])
 
-
     def extract_one_file(self, dataset, image_path, output_file=None):
         """Runs the feature extraction algorithms on specific image.
 

src/extraction/hog.py

@@ -8,6 +8,11 @@
     
     Name: hog.py
     Author: Alessandro dos Santos Ferreira ( santosferreira.alessandro@gmail.com )
+	
+    Change parameter Visualise for Visualize because is deprecaded
+    Date:02/01/2019
+    Author: Diego Andre Sant Ana		 	
+		
 """
 from skimage import feature
 
@@ -36,13 +41,13 @@ class HOG(Extractor):
         features : tuple
             Returns a tuple containing a list of labels, type and values for each feature extracted.
         """
-        image_grayscale = ImageUtils.image_grayscale(image, bgr = True)
+        image_grayscale = ImageUtils.image_grayscale(image, bgr=True)
         image_128x128 = ImageUtils.image_resize(image_grayscale, 128, 128)
-        
+
         values, _ = feature.hog(image_128x128, orientations=8, pixels_per_cell=(32, 32),
-                    cells_per_block=(1, 1), visualise=True)
-                    
-        labels = [m+n for m,n in zip(['hog_'] * len(values),map(str,range(0,len(values))))]
+                                cells_per_block=(1, 1), visualise=True)
+
+        labels = [m + n for m, n in zip(['hog_'] * len(values), map(str, range(0, len(values))))]
         types = [Extractor.NUMERIC] * len(labels)
 
         return labels, types, list(values)

src/extraction/image_moments.py

@@ -16,7 +16,7 @@ import cv2
 from util.utils import ImageUtils
 from skimage.measure import regionprops, moments, moments_central
 from skimage.morphology import label
-
+import numpy as np
 from extractor import Extractor
 
 class RawCentralMoments(Extractor):
@@ -53,8 +53,8 @@ class RawCentralMoments(Extractor):
         
         row = m[0, 1] / m[0, 0]
         col = m[1, 0] / m[0, 0]
-
-        mu = measure.moments_central(image_binary, row, col)
+	
+        mu = measure.moments_central(image_binary, center=(row, col), order=3)
         
         values_mu = [mu[p, q] for (p, q) in self._moments_order]
         labels_mu = [M+str(p)+str(q) for M,(p,q) in zip(['Mu_'] * len(self._moments_order), self._moments_order)]
@@ -104,8 +104,9 @@ class HuMoments(Extractor):
         image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
 
         values_hu= cv2.HuMoments(cv2.moments(image)).flatten()
+        values_hu = list(values_hu)
 
-
+        values_hu= np.nan_to_num(values_hu)
         labels_hu = [m+n for m,n in zip(['Hu_'] * len(values_hu),map(str,range(0,len(values_hu))))]
         
         labels = labels_hu

src/util/split_convertall.sh

+#!/bin/bash
+
+#
+#    Script - convert tif to png
+#
+#    Name: script_convertall.sh
+#    Author: Gabriel Kirsten Menezes (gabriel.kirsten@hotmail.com)
+#
+
+echo "[SCRIPT CONVERT ALL] Initializing..."
+
+dir_train="../../data/demo_split/train"
+dir_validation="../../data/demo_split/validation"
+
+echo "[SCRIPT CONVERT ALL] Converting train..."
+
+for dir_class in `ls $dir_train`;
+do
+    echo "[SCRIPT CONVERT ALL] Converting class -" $dir_class;
+    convert $dir_train/$dir_class/* $dir_train/$dir_class/$dir_class.png
+    echo "[SCRIPT CONVERT ALL] Removing all .tif files in $dir_class ..."
+    rm $dir_train/$dir_class/*.tif
+done
+
+echo "[SCRIPT CONVERT ALL] Converting validation..."
+
+for dir_class in `ls $dir_validation`;
+do
+    echo "[SCRIPT CONVERT ALL] Converting class -" $dir_class;
+    convert $dir_validation/$dir_class/* $dir_validation/$dir_class/$dir_class.png
+    echo "[SCRIPT CONVERT ALL] Removing all .tif files in $dir_class ..."
+    rm $dir_validation/$dir_class/*.tif
+done
+
+echo "[SCRIPT CONVERT ALL] OK! DONE."